1 /*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75 #include <asm/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <net/net_namespace.h>
98 #include <net/sock.h>
99 #include <linux/rtnetlink.h>
100 #include <linux/stat.h>
101 #include <net/dst.h>
102 #include <net/pkt_sched.h>
103 #include <net/checksum.h>
104 #include <net/xfrm.h>
105 #include <linux/highmem.h>
106 #include <linux/init.h>
107 #include <linux/module.h>
108 #include <linux/netpoll.h>
109 #include <linux/rcupdate.h>
110 #include <linux/delay.h>
111 #include <net/iw_handler.h>
112 #include <asm/current.h>
113 #include <linux/audit.h>
114 #include <linux/dmaengine.h>
115 #include <linux/err.h>
116 #include <linux/ctype.h>
117 #include <linux/if_arp.h>
118 #include <linux/if_vlan.h>
119 #include <linux/ip.h>
120 #include <net/ip.h>
121 #include <linux/ipv6.h>
122 #include <linux/in.h>
123 #include <linux/jhash.h>
124 #include <linux/random.h>
125 #include <trace/events/napi.h>
126 #include <trace/events/net.h>
127 #include <trace/events/skb.h>
128 #include <linux/pci.h>
129 #include <linux/inetdevice.h>
130 #include <linux/cpu_rmap.h>
131 #include <linux/static_key.h>
132
133 #include "net-sysfs.h"
134
135 /* Instead of increasing this, you should create a hash table. */
136 #define MAX_GRO_SKBS 8
137
138 /* This should be increased if a protocol with a bigger head is added. */
139 #define GRO_MAX_HEAD (MAX_HEADER + 128)
140
141 static DEFINE_SPINLOCK(ptype_lock);
142 static DEFINE_SPINLOCK(offload_lock);
143 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
144 struct list_head ptype_all __read_mostly; /* Taps */
145 static struct list_head offload_base __read_mostly;
146
147 /*
148 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
149 * semaphore.
150 *
151 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
152 *
153 * Writers must hold the rtnl semaphore while they loop through the
154 * dev_base_head list, and hold dev_base_lock for writing when they do the
155 * actual updates. This allows pure readers to access the list even
156 * while a writer is preparing to update it.
157 *
158 * To put it another way, dev_base_lock is held for writing only to
159 * protect against pure readers; the rtnl semaphore provides the
160 * protection against other writers.
161 *
162 * See, for example usages, register_netdevice() and
163 * unregister_netdevice(), which must be called with the rtnl
164 * semaphore held.
165 */
166 DEFINE_RWLOCK(dev_base_lock);
167 EXPORT_SYMBOL(dev_base_lock);
168
169 seqcount_t devnet_rename_seq;
170
dev_base_seq_inc(struct net * net)171 static inline void dev_base_seq_inc(struct net *net)
172 {
173 while (++net->dev_base_seq == 0);
174 }
175
dev_name_hash(struct net * net,const char * name)176 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
177 {
178 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
179
180 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
181 }
182
dev_index_hash(struct net * net,int ifindex)183 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
184 {
185 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
186 }
187
rps_lock(struct softnet_data * sd)188 static inline void rps_lock(struct softnet_data *sd)
189 {
190 #ifdef CONFIG_RPS
191 spin_lock(&sd->input_pkt_queue.lock);
192 #endif
193 }
194
rps_unlock(struct softnet_data * sd)195 static inline void rps_unlock(struct softnet_data *sd)
196 {
197 #ifdef CONFIG_RPS
198 spin_unlock(&sd->input_pkt_queue.lock);
199 #endif
200 }
201
202 /* Device list insertion */
list_netdevice(struct net_device * dev)203 static void list_netdevice(struct net_device *dev)
204 {
205 struct net *net = dev_net(dev);
206
207 ASSERT_RTNL();
208
209 write_lock_bh(&dev_base_lock);
210 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
211 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
212 hlist_add_head_rcu(&dev->index_hlist,
213 dev_index_hash(net, dev->ifindex));
214 write_unlock_bh(&dev_base_lock);
215
216 dev_base_seq_inc(net);
217 }
218
219 /* Device list removal
220 * caller must respect a RCU grace period before freeing/reusing dev
221 */
unlist_netdevice(struct net_device * dev)222 static void unlist_netdevice(struct net_device *dev)
223 {
224 ASSERT_RTNL();
225
226 /* Unlink dev from the device chain */
227 write_lock_bh(&dev_base_lock);
228 list_del_rcu(&dev->dev_list);
229 hlist_del_rcu(&dev->name_hlist);
230 hlist_del_rcu(&dev->index_hlist);
231 write_unlock_bh(&dev_base_lock);
232
233 dev_base_seq_inc(dev_net(dev));
234 }
235
236 /*
237 * Our notifier list
238 */
239
240 static RAW_NOTIFIER_HEAD(netdev_chain);
241
242 /*
243 * Device drivers call our routines to queue packets here. We empty the
244 * queue in the local softnet handler.
245 */
246
247 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
248 EXPORT_PER_CPU_SYMBOL(softnet_data);
249
250 #ifdef CONFIG_LOCKDEP
251 /*
252 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
253 * according to dev->type
254 */
255 static const unsigned short netdev_lock_type[] =
256 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
257 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
258 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
259 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
260 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
261 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
262 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
263 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
264 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
265 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
266 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
267 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
268 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
269 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
270 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
271
272 static const char *const netdev_lock_name[] =
273 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
274 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
275 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
276 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
277 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
278 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
279 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
280 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
281 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
282 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
283 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
284 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
285 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
286 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
287 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
288
289 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
290 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
291
netdev_lock_pos(unsigned short dev_type)292 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
293 {
294 int i;
295
296 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
297 if (netdev_lock_type[i] == dev_type)
298 return i;
299 /* the last key is used by default */
300 return ARRAY_SIZE(netdev_lock_type) - 1;
301 }
302
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)303 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
304 unsigned short dev_type)
305 {
306 int i;
307
308 i = netdev_lock_pos(dev_type);
309 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
310 netdev_lock_name[i]);
311 }
312
netdev_set_addr_lockdep_class(struct net_device * dev)313 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
314 {
315 int i;
316
317 i = netdev_lock_pos(dev->type);
318 lockdep_set_class_and_name(&dev->addr_list_lock,
319 &netdev_addr_lock_key[i],
320 netdev_lock_name[i]);
321 }
322 #else
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)323 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
324 unsigned short dev_type)
325 {
326 }
netdev_set_addr_lockdep_class(struct net_device * dev)327 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
328 {
329 }
330 #endif
331
332 /*******************************************************************************
333
334 Protocol management and registration routines
335
336 *******************************************************************************/
337
338 /*
339 * Add a protocol ID to the list. Now that the input handler is
340 * smarter we can dispense with all the messy stuff that used to be
341 * here.
342 *
343 * BEWARE!!! Protocol handlers, mangling input packets,
344 * MUST BE last in hash buckets and checking protocol handlers
345 * MUST start from promiscuous ptype_all chain in net_bh.
346 * It is true now, do not change it.
347 * Explanation follows: if protocol handler, mangling packet, will
348 * be the first on list, it is not able to sense, that packet
349 * is cloned and should be copied-on-write, so that it will
350 * change it and subsequent readers will get broken packet.
351 * --ANK (980803)
352 */
353
ptype_head(const struct packet_type * pt)354 static inline struct list_head *ptype_head(const struct packet_type *pt)
355 {
356 if (pt->type == htons(ETH_P_ALL))
357 return &ptype_all;
358 else
359 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
360 }
361
362 /**
363 * dev_add_pack - add packet handler
364 * @pt: packet type declaration
365 *
366 * Add a protocol handler to the networking stack. The passed &packet_type
367 * is linked into kernel lists and may not be freed until it has been
368 * removed from the kernel lists.
369 *
370 * This call does not sleep therefore it can not
371 * guarantee all CPU's that are in middle of receiving packets
372 * will see the new packet type (until the next received packet).
373 */
374
dev_add_pack(struct packet_type * pt)375 void dev_add_pack(struct packet_type *pt)
376 {
377 struct list_head *head = ptype_head(pt);
378
379 spin_lock(&ptype_lock);
380 list_add_rcu(&pt->list, head);
381 spin_unlock(&ptype_lock);
382 }
383 EXPORT_SYMBOL(dev_add_pack);
384
385 /**
386 * __dev_remove_pack - remove packet handler
387 * @pt: packet type declaration
388 *
389 * Remove a protocol handler that was previously added to the kernel
390 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
391 * from the kernel lists and can be freed or reused once this function
392 * returns.
393 *
394 * The packet type might still be in use by receivers
395 * and must not be freed until after all the CPU's have gone
396 * through a quiescent state.
397 */
__dev_remove_pack(struct packet_type * pt)398 void __dev_remove_pack(struct packet_type *pt)
399 {
400 struct list_head *head = ptype_head(pt);
401 struct packet_type *pt1;
402
403 spin_lock(&ptype_lock);
404
405 list_for_each_entry(pt1, head, list) {
406 if (pt == pt1) {
407 list_del_rcu(&pt->list);
408 goto out;
409 }
410 }
411
412 pr_warn("dev_remove_pack: %p not found\n", pt);
413 out:
414 spin_unlock(&ptype_lock);
415 }
416 EXPORT_SYMBOL(__dev_remove_pack);
417
418 /**
419 * dev_remove_pack - remove packet handler
420 * @pt: packet type declaration
421 *
422 * Remove a protocol handler that was previously added to the kernel
423 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
424 * from the kernel lists and can be freed or reused once this function
425 * returns.
426 *
427 * This call sleeps to guarantee that no CPU is looking at the packet
428 * type after return.
429 */
dev_remove_pack(struct packet_type * pt)430 void dev_remove_pack(struct packet_type *pt)
431 {
432 __dev_remove_pack(pt);
433
434 synchronize_net();
435 }
436 EXPORT_SYMBOL(dev_remove_pack);
437
438
439 /**
440 * dev_add_offload - register offload handlers
441 * @po: protocol offload declaration
442 *
443 * Add protocol offload handlers to the networking stack. The passed
444 * &proto_offload is linked into kernel lists and may not be freed until
445 * it has been removed from the kernel lists.
446 *
447 * This call does not sleep therefore it can not
448 * guarantee all CPU's that are in middle of receiving packets
449 * will see the new offload handlers (until the next received packet).
450 */
dev_add_offload(struct packet_offload * po)451 void dev_add_offload(struct packet_offload *po)
452 {
453 struct list_head *head = &offload_base;
454
455 spin_lock(&offload_lock);
456 list_add_rcu(&po->list, head);
457 spin_unlock(&offload_lock);
458 }
459 EXPORT_SYMBOL(dev_add_offload);
460
461 /**
462 * __dev_remove_offload - remove offload handler
463 * @po: packet offload declaration
464 *
465 * Remove a protocol offload handler that was previously added to the
466 * kernel offload handlers by dev_add_offload(). The passed &offload_type
467 * is removed from the kernel lists and can be freed or reused once this
468 * function returns.
469 *
470 * The packet type might still be in use by receivers
471 * and must not be freed until after all the CPU's have gone
472 * through a quiescent state.
473 */
__dev_remove_offload(struct packet_offload * po)474 void __dev_remove_offload(struct packet_offload *po)
475 {
476 struct list_head *head = &offload_base;
477 struct packet_offload *po1;
478
479 spin_lock(&offload_lock);
480
481 list_for_each_entry(po1, head, list) {
482 if (po == po1) {
483 list_del_rcu(&po->list);
484 goto out;
485 }
486 }
487
488 pr_warn("dev_remove_offload: %p not found\n", po);
489 out:
490 spin_unlock(&offload_lock);
491 }
492 EXPORT_SYMBOL(__dev_remove_offload);
493
494 /**
495 * dev_remove_offload - remove packet offload handler
496 * @po: packet offload declaration
497 *
498 * Remove a packet offload handler that was previously added to the kernel
499 * offload handlers by dev_add_offload(). The passed &offload_type is
500 * removed from the kernel lists and can be freed or reused once this
501 * function returns.
502 *
503 * This call sleeps to guarantee that no CPU is looking at the packet
504 * type after return.
505 */
dev_remove_offload(struct packet_offload * po)506 void dev_remove_offload(struct packet_offload *po)
507 {
508 __dev_remove_offload(po);
509
510 synchronize_net();
511 }
512 EXPORT_SYMBOL(dev_remove_offload);
513
514 /******************************************************************************
515
516 Device Boot-time Settings Routines
517
518 *******************************************************************************/
519
520 /* Boot time configuration table */
521 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
522
523 /**
524 * netdev_boot_setup_add - add new setup entry
525 * @name: name of the device
526 * @map: configured settings for the device
527 *
528 * Adds new setup entry to the dev_boot_setup list. The function
529 * returns 0 on error and 1 on success. This is a generic routine to
530 * all netdevices.
531 */
netdev_boot_setup_add(char * name,struct ifmap * map)532 static int netdev_boot_setup_add(char *name, struct ifmap *map)
533 {
534 struct netdev_boot_setup *s;
535 int i;
536
537 s = dev_boot_setup;
538 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
539 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
540 memset(s[i].name, 0, sizeof(s[i].name));
541 strlcpy(s[i].name, name, IFNAMSIZ);
542 memcpy(&s[i].map, map, sizeof(s[i].map));
543 break;
544 }
545 }
546
547 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
548 }
549
550 /**
551 * netdev_boot_setup_check - check boot time settings
552 * @dev: the netdevice
553 *
554 * Check boot time settings for the device.
555 * The found settings are set for the device to be used
556 * later in the device probing.
557 * Returns 0 if no settings found, 1 if they are.
558 */
netdev_boot_setup_check(struct net_device * dev)559 int netdev_boot_setup_check(struct net_device *dev)
560 {
561 struct netdev_boot_setup *s = dev_boot_setup;
562 int i;
563
564 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
565 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
566 !strcmp(dev->name, s[i].name)) {
567 dev->irq = s[i].map.irq;
568 dev->base_addr = s[i].map.base_addr;
569 dev->mem_start = s[i].map.mem_start;
570 dev->mem_end = s[i].map.mem_end;
571 return 1;
572 }
573 }
574 return 0;
575 }
576 EXPORT_SYMBOL(netdev_boot_setup_check);
577
578
579 /**
580 * netdev_boot_base - get address from boot time settings
581 * @prefix: prefix for network device
582 * @unit: id for network device
583 *
584 * Check boot time settings for the base address of device.
585 * The found settings are set for the device to be used
586 * later in the device probing.
587 * Returns 0 if no settings found.
588 */
netdev_boot_base(const char * prefix,int unit)589 unsigned long netdev_boot_base(const char *prefix, int unit)
590 {
591 const struct netdev_boot_setup *s = dev_boot_setup;
592 char name[IFNAMSIZ];
593 int i;
594
595 sprintf(name, "%s%d", prefix, unit);
596
597 /*
598 * If device already registered then return base of 1
599 * to indicate not to probe for this interface
600 */
601 if (__dev_get_by_name(&init_net, name))
602 return 1;
603
604 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
605 if (!strcmp(name, s[i].name))
606 return s[i].map.base_addr;
607 return 0;
608 }
609
610 /*
611 * Saves at boot time configured settings for any netdevice.
612 */
netdev_boot_setup(char * str)613 int __init netdev_boot_setup(char *str)
614 {
615 int ints[5];
616 struct ifmap map;
617
618 str = get_options(str, ARRAY_SIZE(ints), ints);
619 if (!str || !*str)
620 return 0;
621
622 /* Save settings */
623 memset(&map, 0, sizeof(map));
624 if (ints[0] > 0)
625 map.irq = ints[1];
626 if (ints[0] > 1)
627 map.base_addr = ints[2];
628 if (ints[0] > 2)
629 map.mem_start = ints[3];
630 if (ints[0] > 3)
631 map.mem_end = ints[4];
632
633 /* Add new entry to the list */
634 return netdev_boot_setup_add(str, &map);
635 }
636
637 __setup("netdev=", netdev_boot_setup);
638
639 /*******************************************************************************
640
641 Device Interface Subroutines
642
643 *******************************************************************************/
644
645 /**
646 * __dev_get_by_name - find a device by its name
647 * @net: the applicable net namespace
648 * @name: name to find
649 *
650 * Find an interface by name. Must be called under RTNL semaphore
651 * or @dev_base_lock. If the name is found a pointer to the device
652 * is returned. If the name is not found then %NULL is returned. The
653 * reference counters are not incremented so the caller must be
654 * careful with locks.
655 */
656
__dev_get_by_name(struct net * net,const char * name)657 struct net_device *__dev_get_by_name(struct net *net, const char *name)
658 {
659 struct net_device *dev;
660 struct hlist_head *head = dev_name_hash(net, name);
661
662 hlist_for_each_entry(dev, head, name_hlist)
663 if (!strncmp(dev->name, name, IFNAMSIZ))
664 return dev;
665
666 return NULL;
667 }
668 EXPORT_SYMBOL(__dev_get_by_name);
669
670 /**
671 * dev_get_by_name_rcu - find a device by its name
672 * @net: the applicable net namespace
673 * @name: name to find
674 *
675 * Find an interface by name.
676 * If the name is found a pointer to the device is returned.
677 * If the name is not found then %NULL is returned.
678 * The reference counters are not incremented so the caller must be
679 * careful with locks. The caller must hold RCU lock.
680 */
681
dev_get_by_name_rcu(struct net * net,const char * name)682 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
683 {
684 struct net_device *dev;
685 struct hlist_head *head = dev_name_hash(net, name);
686
687 hlist_for_each_entry_rcu(dev, head, name_hlist)
688 if (!strncmp(dev->name, name, IFNAMSIZ))
689 return dev;
690
691 return NULL;
692 }
693 EXPORT_SYMBOL(dev_get_by_name_rcu);
694
695 /**
696 * dev_get_by_name - find a device by its name
697 * @net: the applicable net namespace
698 * @name: name to find
699 *
700 * Find an interface by name. This can be called from any
701 * context and does its own locking. The returned handle has
702 * the usage count incremented and the caller must use dev_put() to
703 * release it when it is no longer needed. %NULL is returned if no
704 * matching device is found.
705 */
706
dev_get_by_name(struct net * net,const char * name)707 struct net_device *dev_get_by_name(struct net *net, const char *name)
708 {
709 struct net_device *dev;
710
711 rcu_read_lock();
712 dev = dev_get_by_name_rcu(net, name);
713 if (dev)
714 dev_hold(dev);
715 rcu_read_unlock();
716 return dev;
717 }
718 EXPORT_SYMBOL(dev_get_by_name);
719
720 /**
721 * __dev_get_by_index - find a device by its ifindex
722 * @net: the applicable net namespace
723 * @ifindex: index of device
724 *
725 * Search for an interface by index. Returns %NULL if the device
726 * is not found or a pointer to the device. The device has not
727 * had its reference counter increased so the caller must be careful
728 * about locking. The caller must hold either the RTNL semaphore
729 * or @dev_base_lock.
730 */
731
__dev_get_by_index(struct net * net,int ifindex)732 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
733 {
734 struct net_device *dev;
735 struct hlist_head *head = dev_index_hash(net, ifindex);
736
737 hlist_for_each_entry(dev, head, index_hlist)
738 if (dev->ifindex == ifindex)
739 return dev;
740
741 return NULL;
742 }
743 EXPORT_SYMBOL(__dev_get_by_index);
744
745 /**
746 * dev_get_by_index_rcu - find a device by its ifindex
747 * @net: the applicable net namespace
748 * @ifindex: index of device
749 *
750 * Search for an interface by index. Returns %NULL if the device
751 * is not found or a pointer to the device. The device has not
752 * had its reference counter increased so the caller must be careful
753 * about locking. The caller must hold RCU lock.
754 */
755
dev_get_by_index_rcu(struct net * net,int ifindex)756 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
757 {
758 struct net_device *dev;
759 struct hlist_head *head = dev_index_hash(net, ifindex);
760
761 hlist_for_each_entry_rcu(dev, head, index_hlist)
762 if (dev->ifindex == ifindex)
763 return dev;
764
765 return NULL;
766 }
767 EXPORT_SYMBOL(dev_get_by_index_rcu);
768
769
770 /**
771 * dev_get_by_index - find a device by its ifindex
772 * @net: the applicable net namespace
773 * @ifindex: index of device
774 *
775 * Search for an interface by index. Returns NULL if the device
776 * is not found or a pointer to the device. The device returned has
777 * had a reference added and the pointer is safe until the user calls
778 * dev_put to indicate they have finished with it.
779 */
780
dev_get_by_index(struct net * net,int ifindex)781 struct net_device *dev_get_by_index(struct net *net, int ifindex)
782 {
783 struct net_device *dev;
784
785 rcu_read_lock();
786 dev = dev_get_by_index_rcu(net, ifindex);
787 if (dev)
788 dev_hold(dev);
789 rcu_read_unlock();
790 return dev;
791 }
792 EXPORT_SYMBOL(dev_get_by_index);
793
794 /**
795 * netdev_get_name - get a netdevice name, knowing its ifindex.
796 * @net: network namespace
797 * @name: a pointer to the buffer where the name will be stored.
798 * @ifindex: the ifindex of the interface to get the name from.
799 *
800 * The use of raw_seqcount_begin() and cond_resched() before
801 * retrying is required as we want to give the writers a chance
802 * to complete when CONFIG_PREEMPT is not set.
803 */
netdev_get_name(struct net * net,char * name,int ifindex)804 int netdev_get_name(struct net *net, char *name, int ifindex)
805 {
806 struct net_device *dev;
807 unsigned int seq;
808
809 retry:
810 seq = raw_seqcount_begin(&devnet_rename_seq);
811 rcu_read_lock();
812 dev = dev_get_by_index_rcu(net, ifindex);
813 if (!dev) {
814 rcu_read_unlock();
815 return -ENODEV;
816 }
817
818 strcpy(name, dev->name);
819 rcu_read_unlock();
820 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
821 cond_resched();
822 goto retry;
823 }
824
825 return 0;
826 }
827
828 /**
829 * dev_getbyhwaddr_rcu - find a device by its hardware address
830 * @net: the applicable net namespace
831 * @type: media type of device
832 * @ha: hardware address
833 *
834 * Search for an interface by MAC address. Returns NULL if the device
835 * is not found or a pointer to the device.
836 * The caller must hold RCU or RTNL.
837 * The returned device has not had its ref count increased
838 * and the caller must therefore be careful about locking
839 *
840 */
841
dev_getbyhwaddr_rcu(struct net * net,unsigned short type,const char * ha)842 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
843 const char *ha)
844 {
845 struct net_device *dev;
846
847 for_each_netdev_rcu(net, dev)
848 if (dev->type == type &&
849 !memcmp(dev->dev_addr, ha, dev->addr_len))
850 return dev;
851
852 return NULL;
853 }
854 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
855
__dev_getfirstbyhwtype(struct net * net,unsigned short type)856 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
857 {
858 struct net_device *dev;
859
860 ASSERT_RTNL();
861 for_each_netdev(net, dev)
862 if (dev->type == type)
863 return dev;
864
865 return NULL;
866 }
867 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
868
dev_getfirstbyhwtype(struct net * net,unsigned short type)869 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
870 {
871 struct net_device *dev, *ret = NULL;
872
873 rcu_read_lock();
874 for_each_netdev_rcu(net, dev)
875 if (dev->type == type) {
876 dev_hold(dev);
877 ret = dev;
878 break;
879 }
880 rcu_read_unlock();
881 return ret;
882 }
883 EXPORT_SYMBOL(dev_getfirstbyhwtype);
884
885 /**
886 * dev_get_by_flags_rcu - find any device with given flags
887 * @net: the applicable net namespace
888 * @if_flags: IFF_* values
889 * @mask: bitmask of bits in if_flags to check
890 *
891 * Search for any interface with the given flags. Returns NULL if a device
892 * is not found or a pointer to the device. Must be called inside
893 * rcu_read_lock(), and result refcount is unchanged.
894 */
895
dev_get_by_flags_rcu(struct net * net,unsigned short if_flags,unsigned short mask)896 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
897 unsigned short mask)
898 {
899 struct net_device *dev, *ret;
900
901 ret = NULL;
902 for_each_netdev_rcu(net, dev) {
903 if (((dev->flags ^ if_flags) & mask) == 0) {
904 ret = dev;
905 break;
906 }
907 }
908 return ret;
909 }
910 EXPORT_SYMBOL(dev_get_by_flags_rcu);
911
912 /**
913 * dev_valid_name - check if name is okay for network device
914 * @name: name string
915 *
916 * Network device names need to be valid file names to
917 * to allow sysfs to work. We also disallow any kind of
918 * whitespace.
919 */
dev_valid_name(const char * name)920 bool dev_valid_name(const char *name)
921 {
922 if (*name == '\0')
923 return false;
924 if (strlen(name) >= IFNAMSIZ)
925 return false;
926 if (!strcmp(name, ".") || !strcmp(name, ".."))
927 return false;
928
929 while (*name) {
930 if (*name == '/' || isspace(*name))
931 return false;
932 name++;
933 }
934 return true;
935 }
936 EXPORT_SYMBOL(dev_valid_name);
937
938 /**
939 * __dev_alloc_name - allocate a name for a device
940 * @net: network namespace to allocate the device name in
941 * @name: name format string
942 * @buf: scratch buffer and result name string
943 *
944 * Passed a format string - eg "lt%d" it will try and find a suitable
945 * id. It scans list of devices to build up a free map, then chooses
946 * the first empty slot. The caller must hold the dev_base or rtnl lock
947 * while allocating the name and adding the device in order to avoid
948 * duplicates.
949 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
950 * Returns the number of the unit assigned or a negative errno code.
951 */
952
__dev_alloc_name(struct net * net,const char * name,char * buf)953 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
954 {
955 int i = 0;
956 const char *p;
957 const int max_netdevices = 8*PAGE_SIZE;
958 unsigned long *inuse;
959 struct net_device *d;
960
961 p = strnchr(name, IFNAMSIZ-1, '%');
962 if (p) {
963 /*
964 * Verify the string as this thing may have come from
965 * the user. There must be either one "%d" and no other "%"
966 * characters.
967 */
968 if (p[1] != 'd' || strchr(p + 2, '%'))
969 return -EINVAL;
970
971 /* Use one page as a bit array of possible slots */
972 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
973 if (!inuse)
974 return -ENOMEM;
975
976 for_each_netdev(net, d) {
977 if (!sscanf(d->name, name, &i))
978 continue;
979 if (i < 0 || i >= max_netdevices)
980 continue;
981
982 /* avoid cases where sscanf is not exact inverse of printf */
983 snprintf(buf, IFNAMSIZ, name, i);
984 if (!strncmp(buf, d->name, IFNAMSIZ))
985 set_bit(i, inuse);
986 }
987
988 i = find_first_zero_bit(inuse, max_netdevices);
989 free_page((unsigned long) inuse);
990 }
991
992 if (buf != name)
993 snprintf(buf, IFNAMSIZ, name, i);
994 if (!__dev_get_by_name(net, buf))
995 return i;
996
997 /* It is possible to run out of possible slots
998 * when the name is long and there isn't enough space left
999 * for the digits, or if all bits are used.
1000 */
1001 return -ENFILE;
1002 }
1003
1004 /**
1005 * dev_alloc_name - allocate a name for a device
1006 * @dev: device
1007 * @name: name format string
1008 *
1009 * Passed a format string - eg "lt%d" it will try and find a suitable
1010 * id. It scans list of devices to build up a free map, then chooses
1011 * the first empty slot. The caller must hold the dev_base or rtnl lock
1012 * while allocating the name and adding the device in order to avoid
1013 * duplicates.
1014 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1015 * Returns the number of the unit assigned or a negative errno code.
1016 */
1017
dev_alloc_name(struct net_device * dev,const char * name)1018 int dev_alloc_name(struct net_device *dev, const char *name)
1019 {
1020 char buf[IFNAMSIZ];
1021 struct net *net;
1022 int ret;
1023
1024 BUG_ON(!dev_net(dev));
1025 net = dev_net(dev);
1026 ret = __dev_alloc_name(net, name, buf);
1027 if (ret >= 0)
1028 strlcpy(dev->name, buf, IFNAMSIZ);
1029 return ret;
1030 }
1031 EXPORT_SYMBOL(dev_alloc_name);
1032
dev_alloc_name_ns(struct net * net,struct net_device * dev,const char * name)1033 static int dev_alloc_name_ns(struct net *net,
1034 struct net_device *dev,
1035 const char *name)
1036 {
1037 char buf[IFNAMSIZ];
1038 int ret;
1039
1040 ret = __dev_alloc_name(net, name, buf);
1041 if (ret >= 0)
1042 strlcpy(dev->name, buf, IFNAMSIZ);
1043 return ret;
1044 }
1045
dev_get_valid_name(struct net * net,struct net_device * dev,const char * name)1046 static int dev_get_valid_name(struct net *net,
1047 struct net_device *dev,
1048 const char *name)
1049 {
1050 BUG_ON(!net);
1051
1052 if (!dev_valid_name(name))
1053 return -EINVAL;
1054
1055 if (strchr(name, '%'))
1056 return dev_alloc_name_ns(net, dev, name);
1057 else if (__dev_get_by_name(net, name))
1058 return -EEXIST;
1059 else if (dev->name != name)
1060 strlcpy(dev->name, name, IFNAMSIZ);
1061
1062 return 0;
1063 }
1064
1065 /**
1066 * dev_change_name - change name of a device
1067 * @dev: device
1068 * @newname: name (or format string) must be at least IFNAMSIZ
1069 *
1070 * Change name of a device, can pass format strings "eth%d".
1071 * for wildcarding.
1072 */
dev_change_name(struct net_device * dev,const char * newname)1073 int dev_change_name(struct net_device *dev, const char *newname)
1074 {
1075 char oldname[IFNAMSIZ];
1076 int err = 0;
1077 int ret;
1078 struct net *net;
1079
1080 ASSERT_RTNL();
1081 BUG_ON(!dev_net(dev));
1082
1083 net = dev_net(dev);
1084 if (dev->flags & IFF_UP)
1085 return -EBUSY;
1086
1087 write_seqcount_begin(&devnet_rename_seq);
1088
1089 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1090 write_seqcount_end(&devnet_rename_seq);
1091 return 0;
1092 }
1093
1094 memcpy(oldname, dev->name, IFNAMSIZ);
1095
1096 err = dev_get_valid_name(net, dev, newname);
1097 if (err < 0) {
1098 write_seqcount_end(&devnet_rename_seq);
1099 return err;
1100 }
1101
1102 rollback:
1103 ret = device_rename(&dev->dev, dev->name);
1104 if (ret) {
1105 memcpy(dev->name, oldname, IFNAMSIZ);
1106 write_seqcount_end(&devnet_rename_seq);
1107 return ret;
1108 }
1109
1110 write_seqcount_end(&devnet_rename_seq);
1111
1112 write_lock_bh(&dev_base_lock);
1113 hlist_del_rcu(&dev->name_hlist);
1114 write_unlock_bh(&dev_base_lock);
1115
1116 synchronize_rcu();
1117
1118 write_lock_bh(&dev_base_lock);
1119 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1120 write_unlock_bh(&dev_base_lock);
1121
1122 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1123 ret = notifier_to_errno(ret);
1124
1125 if (ret) {
1126 /* err >= 0 after dev_alloc_name() or stores the first errno */
1127 if (err >= 0) {
1128 err = ret;
1129 write_seqcount_begin(&devnet_rename_seq);
1130 memcpy(dev->name, oldname, IFNAMSIZ);
1131 goto rollback;
1132 } else {
1133 pr_err("%s: name change rollback failed: %d\n",
1134 dev->name, ret);
1135 }
1136 }
1137
1138 return err;
1139 }
1140
1141 /**
1142 * dev_set_alias - change ifalias of a device
1143 * @dev: device
1144 * @alias: name up to IFALIASZ
1145 * @len: limit of bytes to copy from info
1146 *
1147 * Set ifalias for a device,
1148 */
dev_set_alias(struct net_device * dev,const char * alias,size_t len)1149 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1150 {
1151 char *new_ifalias;
1152
1153 ASSERT_RTNL();
1154
1155 if (len >= IFALIASZ)
1156 return -EINVAL;
1157
1158 if (!len) {
1159 kfree(dev->ifalias);
1160 dev->ifalias = NULL;
1161 return 0;
1162 }
1163
1164 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1165 if (!new_ifalias)
1166 return -ENOMEM;
1167 dev->ifalias = new_ifalias;
1168
1169 strlcpy(dev->ifalias, alias, len+1);
1170 return len;
1171 }
1172
1173
1174 /**
1175 * netdev_features_change - device changes features
1176 * @dev: device to cause notification
1177 *
1178 * Called to indicate a device has changed features.
1179 */
netdev_features_change(struct net_device * dev)1180 void netdev_features_change(struct net_device *dev)
1181 {
1182 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1183 }
1184 EXPORT_SYMBOL(netdev_features_change);
1185
1186 /**
1187 * netdev_state_change - device changes state
1188 * @dev: device to cause notification
1189 *
1190 * Called to indicate a device has changed state. This function calls
1191 * the notifier chains for netdev_chain and sends a NEWLINK message
1192 * to the routing socket.
1193 */
netdev_state_change(struct net_device * dev)1194 void netdev_state_change(struct net_device *dev)
1195 {
1196 if (dev->flags & IFF_UP) {
1197 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1198 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1199 }
1200 }
1201 EXPORT_SYMBOL(netdev_state_change);
1202
1203 /**
1204 * netdev_notify_peers - notify network peers about existence of @dev
1205 * @dev: network device
1206 *
1207 * Generate traffic such that interested network peers are aware of
1208 * @dev, such as by generating a gratuitous ARP. This may be used when
1209 * a device wants to inform the rest of the network about some sort of
1210 * reconfiguration such as a failover event or virtual machine
1211 * migration.
1212 */
netdev_notify_peers(struct net_device * dev)1213 void netdev_notify_peers(struct net_device *dev)
1214 {
1215 rtnl_lock();
1216 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1217 rtnl_unlock();
1218 }
1219 EXPORT_SYMBOL(netdev_notify_peers);
1220
__dev_open(struct net_device * dev)1221 static int __dev_open(struct net_device *dev)
1222 {
1223 const struct net_device_ops *ops = dev->netdev_ops;
1224 int ret;
1225
1226 ASSERT_RTNL();
1227
1228 if (!netif_device_present(dev))
1229 return -ENODEV;
1230
1231 /* Block netpoll from trying to do any rx path servicing.
1232 * If we don't do this there is a chance ndo_poll_controller
1233 * or ndo_poll may be running while we open the device
1234 */
1235 ret = netpoll_rx_disable(dev);
1236 if (ret)
1237 return ret;
1238
1239 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1240 ret = notifier_to_errno(ret);
1241 if (ret)
1242 return ret;
1243
1244 set_bit(__LINK_STATE_START, &dev->state);
1245
1246 if (ops->ndo_validate_addr)
1247 ret = ops->ndo_validate_addr(dev);
1248
1249 if (!ret && ops->ndo_open)
1250 ret = ops->ndo_open(dev);
1251
1252 netpoll_rx_enable(dev);
1253
1254 if (ret)
1255 clear_bit(__LINK_STATE_START, &dev->state);
1256 else {
1257 dev->flags |= IFF_UP;
1258 net_dmaengine_get();
1259 dev_set_rx_mode(dev);
1260 dev_activate(dev);
1261 add_device_randomness(dev->dev_addr, dev->addr_len);
1262 }
1263
1264 return ret;
1265 }
1266
1267 /**
1268 * dev_open - prepare an interface for use.
1269 * @dev: device to open
1270 *
1271 * Takes a device from down to up state. The device's private open
1272 * function is invoked and then the multicast lists are loaded. Finally
1273 * the device is moved into the up state and a %NETDEV_UP message is
1274 * sent to the netdev notifier chain.
1275 *
1276 * Calling this function on an active interface is a nop. On a failure
1277 * a negative errno code is returned.
1278 */
dev_open(struct net_device * dev)1279 int dev_open(struct net_device *dev)
1280 {
1281 int ret;
1282
1283 if (dev->flags & IFF_UP)
1284 return 0;
1285
1286 ret = __dev_open(dev);
1287 if (ret < 0)
1288 return ret;
1289
1290 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1291 call_netdevice_notifiers(NETDEV_UP, dev);
1292
1293 return ret;
1294 }
1295 EXPORT_SYMBOL(dev_open);
1296
__dev_close_many(struct list_head * head)1297 static int __dev_close_many(struct list_head *head)
1298 {
1299 struct net_device *dev;
1300
1301 ASSERT_RTNL();
1302 might_sleep();
1303
1304 list_for_each_entry(dev, head, unreg_list) {
1305 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1306
1307 clear_bit(__LINK_STATE_START, &dev->state);
1308
1309 /* Synchronize to scheduled poll. We cannot touch poll list, it
1310 * can be even on different cpu. So just clear netif_running().
1311 *
1312 * dev->stop() will invoke napi_disable() on all of it's
1313 * napi_struct instances on this device.
1314 */
1315 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1316 }
1317
1318 dev_deactivate_many(head);
1319
1320 list_for_each_entry(dev, head, unreg_list) {
1321 const struct net_device_ops *ops = dev->netdev_ops;
1322
1323 /*
1324 * Call the device specific close. This cannot fail.
1325 * Only if device is UP
1326 *
1327 * We allow it to be called even after a DETACH hot-plug
1328 * event.
1329 */
1330 if (ops->ndo_stop)
1331 ops->ndo_stop(dev);
1332
1333 dev->flags &= ~IFF_UP;
1334 net_dmaengine_put();
1335 }
1336
1337 return 0;
1338 }
1339
__dev_close(struct net_device * dev)1340 static int __dev_close(struct net_device *dev)
1341 {
1342 int retval;
1343 LIST_HEAD(single);
1344
1345 /* Temporarily disable netpoll until the interface is down */
1346 retval = netpoll_rx_disable(dev);
1347 if (retval)
1348 return retval;
1349
1350 list_add(&dev->unreg_list, &single);
1351 retval = __dev_close_many(&single);
1352 list_del(&single);
1353
1354 netpoll_rx_enable(dev);
1355 return retval;
1356 }
1357
dev_close_many(struct list_head * head)1358 static int dev_close_many(struct list_head *head)
1359 {
1360 struct net_device *dev, *tmp;
1361 LIST_HEAD(tmp_list);
1362
1363 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1364 if (!(dev->flags & IFF_UP))
1365 list_move(&dev->unreg_list, &tmp_list);
1366
1367 __dev_close_many(head);
1368
1369 list_for_each_entry(dev, head, unreg_list) {
1370 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1371 call_netdevice_notifiers(NETDEV_DOWN, dev);
1372 }
1373
1374 /* rollback_registered_many needs the complete original list */
1375 list_splice(&tmp_list, head);
1376 return 0;
1377 }
1378
1379 /**
1380 * dev_close - shutdown an interface.
1381 * @dev: device to shutdown
1382 *
1383 * This function moves an active device into down state. A
1384 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1385 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1386 * chain.
1387 */
dev_close(struct net_device * dev)1388 int dev_close(struct net_device *dev)
1389 {
1390 int ret = 0;
1391 if (dev->flags & IFF_UP) {
1392 LIST_HEAD(single);
1393
1394 /* Block netpoll rx while the interface is going down */
1395 ret = netpoll_rx_disable(dev);
1396 if (ret)
1397 return ret;
1398
1399 list_add(&dev->unreg_list, &single);
1400 dev_close_many(&single);
1401 list_del(&single);
1402
1403 netpoll_rx_enable(dev);
1404 }
1405 return ret;
1406 }
1407 EXPORT_SYMBOL(dev_close);
1408
1409
1410 /**
1411 * dev_disable_lro - disable Large Receive Offload on a device
1412 * @dev: device
1413 *
1414 * Disable Large Receive Offload (LRO) on a net device. Must be
1415 * called under RTNL. This is needed if received packets may be
1416 * forwarded to another interface.
1417 */
dev_disable_lro(struct net_device * dev)1418 void dev_disable_lro(struct net_device *dev)
1419 {
1420 /*
1421 * If we're trying to disable lro on a vlan device
1422 * use the underlying physical device instead
1423 */
1424 if (is_vlan_dev(dev))
1425 dev = vlan_dev_real_dev(dev);
1426
1427 dev->wanted_features &= ~NETIF_F_LRO;
1428 netdev_update_features(dev);
1429
1430 if (unlikely(dev->features & NETIF_F_LRO))
1431 netdev_WARN(dev, "failed to disable LRO!\n");
1432 }
1433 EXPORT_SYMBOL(dev_disable_lro);
1434
1435
1436 static int dev_boot_phase = 1;
1437
1438 /**
1439 * register_netdevice_notifier - register a network notifier block
1440 * @nb: notifier
1441 *
1442 * Register a notifier to be called when network device events occur.
1443 * The notifier passed is linked into the kernel structures and must
1444 * not be reused until it has been unregistered. A negative errno code
1445 * is returned on a failure.
1446 *
1447 * When registered all registration and up events are replayed
1448 * to the new notifier to allow device to have a race free
1449 * view of the network device list.
1450 */
1451
register_netdevice_notifier(struct notifier_block * nb)1452 int register_netdevice_notifier(struct notifier_block *nb)
1453 {
1454 struct net_device *dev;
1455 struct net_device *last;
1456 struct net *net;
1457 int err;
1458
1459 rtnl_lock();
1460 err = raw_notifier_chain_register(&netdev_chain, nb);
1461 if (err)
1462 goto unlock;
1463 if (dev_boot_phase)
1464 goto unlock;
1465 for_each_net(net) {
1466 for_each_netdev(net, dev) {
1467 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1468 err = notifier_to_errno(err);
1469 if (err)
1470 goto rollback;
1471
1472 if (!(dev->flags & IFF_UP))
1473 continue;
1474
1475 nb->notifier_call(nb, NETDEV_UP, dev);
1476 }
1477 }
1478
1479 unlock:
1480 rtnl_unlock();
1481 return err;
1482
1483 rollback:
1484 last = dev;
1485 for_each_net(net) {
1486 for_each_netdev(net, dev) {
1487 if (dev == last)
1488 goto outroll;
1489
1490 if (dev->flags & IFF_UP) {
1491 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1492 nb->notifier_call(nb, NETDEV_DOWN, dev);
1493 }
1494 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1495 }
1496 }
1497
1498 outroll:
1499 raw_notifier_chain_unregister(&netdev_chain, nb);
1500 goto unlock;
1501 }
1502 EXPORT_SYMBOL(register_netdevice_notifier);
1503
1504 /**
1505 * unregister_netdevice_notifier - unregister a network notifier block
1506 * @nb: notifier
1507 *
1508 * Unregister a notifier previously registered by
1509 * register_netdevice_notifier(). The notifier is unlinked into the
1510 * kernel structures and may then be reused. A negative errno code
1511 * is returned on a failure.
1512 *
1513 * After unregistering unregister and down device events are synthesized
1514 * for all devices on the device list to the removed notifier to remove
1515 * the need for special case cleanup code.
1516 */
1517
unregister_netdevice_notifier(struct notifier_block * nb)1518 int unregister_netdevice_notifier(struct notifier_block *nb)
1519 {
1520 struct net_device *dev;
1521 struct net *net;
1522 int err;
1523
1524 rtnl_lock();
1525 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1526 if (err)
1527 goto unlock;
1528
1529 for_each_net(net) {
1530 for_each_netdev(net, dev) {
1531 if (dev->flags & IFF_UP) {
1532 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1533 nb->notifier_call(nb, NETDEV_DOWN, dev);
1534 }
1535 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1536 }
1537 }
1538 unlock:
1539 rtnl_unlock();
1540 return err;
1541 }
1542 EXPORT_SYMBOL(unregister_netdevice_notifier);
1543
1544 /**
1545 * call_netdevice_notifiers - call all network notifier blocks
1546 * @val: value passed unmodified to notifier function
1547 * @dev: net_device pointer passed unmodified to notifier function
1548 *
1549 * Call all network notifier blocks. Parameters and return value
1550 * are as for raw_notifier_call_chain().
1551 */
1552
call_netdevice_notifiers(unsigned long val,struct net_device * dev)1553 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1554 {
1555 ASSERT_RTNL();
1556 return raw_notifier_call_chain(&netdev_chain, val, dev);
1557 }
1558 EXPORT_SYMBOL(call_netdevice_notifiers);
1559
1560 static struct static_key netstamp_needed __read_mostly;
1561 #ifdef HAVE_JUMP_LABEL
1562 /* We are not allowed to call static_key_slow_dec() from irq context
1563 * If net_disable_timestamp() is called from irq context, defer the
1564 * static_key_slow_dec() calls.
1565 */
1566 static atomic_t netstamp_needed_deferred;
1567 #endif
1568
net_enable_timestamp(void)1569 void net_enable_timestamp(void)
1570 {
1571 #ifdef HAVE_JUMP_LABEL
1572 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1573
1574 if (deferred) {
1575 while (--deferred)
1576 static_key_slow_dec(&netstamp_needed);
1577 return;
1578 }
1579 #endif
1580 static_key_slow_inc(&netstamp_needed);
1581 }
1582 EXPORT_SYMBOL(net_enable_timestamp);
1583
net_disable_timestamp(void)1584 void net_disable_timestamp(void)
1585 {
1586 #ifdef HAVE_JUMP_LABEL
1587 if (in_interrupt()) {
1588 atomic_inc(&netstamp_needed_deferred);
1589 return;
1590 }
1591 #endif
1592 static_key_slow_dec(&netstamp_needed);
1593 }
1594 EXPORT_SYMBOL(net_disable_timestamp);
1595
net_timestamp_set(struct sk_buff * skb)1596 static inline void net_timestamp_set(struct sk_buff *skb)
1597 {
1598 skb->tstamp.tv64 = 0;
1599 if (static_key_false(&netstamp_needed))
1600 __net_timestamp(skb);
1601 }
1602
1603 #define net_timestamp_check(COND, SKB) \
1604 if (static_key_false(&netstamp_needed)) { \
1605 if ((COND) && !(SKB)->tstamp.tv64) \
1606 __net_timestamp(SKB); \
1607 } \
1608
is_skb_forwardable(struct net_device * dev,struct sk_buff * skb)1609 static inline bool is_skb_forwardable(struct net_device *dev,
1610 struct sk_buff *skb)
1611 {
1612 unsigned int len;
1613
1614 if (!(dev->flags & IFF_UP))
1615 return false;
1616
1617 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1618 if (skb->len <= len)
1619 return true;
1620
1621 /* if TSO is enabled, we don't care about the length as the packet
1622 * could be forwarded without being segmented before
1623 */
1624 if (skb_is_gso(skb))
1625 return true;
1626
1627 return false;
1628 }
1629
1630 /**
1631 * dev_forward_skb - loopback an skb to another netif
1632 *
1633 * @dev: destination network device
1634 * @skb: buffer to forward
1635 *
1636 * return values:
1637 * NET_RX_SUCCESS (no congestion)
1638 * NET_RX_DROP (packet was dropped, but freed)
1639 *
1640 * dev_forward_skb can be used for injecting an skb from the
1641 * start_xmit function of one device into the receive queue
1642 * of another device.
1643 *
1644 * The receiving device may be in another namespace, so
1645 * we have to clear all information in the skb that could
1646 * impact namespace isolation.
1647 */
dev_forward_skb(struct net_device * dev,struct sk_buff * skb)1648 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1649 {
1650 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1651 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1652 atomic_long_inc(&dev->rx_dropped);
1653 kfree_skb(skb);
1654 return NET_RX_DROP;
1655 }
1656 }
1657
1658 skb_orphan(skb);
1659
1660 if (unlikely(!is_skb_forwardable(dev, skb))) {
1661 atomic_long_inc(&dev->rx_dropped);
1662 kfree_skb(skb);
1663 return NET_RX_DROP;
1664 }
1665 skb->skb_iif = 0;
1666 skb->dev = dev;
1667 skb_dst_drop(skb);
1668 skb->tstamp.tv64 = 0;
1669 skb->pkt_type = PACKET_HOST;
1670 skb->protocol = eth_type_trans(skb, dev);
1671 skb->mark = 0;
1672 secpath_reset(skb);
1673 nf_reset(skb);
1674 nf_reset_trace(skb);
1675 return netif_rx(skb);
1676 }
1677 EXPORT_SYMBOL_GPL(dev_forward_skb);
1678
deliver_skb(struct sk_buff * skb,struct packet_type * pt_prev,struct net_device * orig_dev)1679 static inline int deliver_skb(struct sk_buff *skb,
1680 struct packet_type *pt_prev,
1681 struct net_device *orig_dev)
1682 {
1683 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1684 return -ENOMEM;
1685 atomic_inc(&skb->users);
1686 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1687 }
1688
skb_loop_sk(struct packet_type * ptype,struct sk_buff * skb)1689 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1690 {
1691 if (!ptype->af_packet_priv || !skb->sk)
1692 return false;
1693
1694 if (ptype->id_match)
1695 return ptype->id_match(ptype, skb->sk);
1696 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1697 return true;
1698
1699 return false;
1700 }
1701
1702 /*
1703 * Support routine. Sends outgoing frames to any network
1704 * taps currently in use.
1705 */
1706
dev_queue_xmit_nit(struct sk_buff * skb,struct net_device * dev)1707 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1708 {
1709 struct packet_type *ptype;
1710 struct sk_buff *skb2 = NULL;
1711 struct packet_type *pt_prev = NULL;
1712
1713 rcu_read_lock();
1714 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1715 /* Never send packets back to the socket
1716 * they originated from - MvS (miquels@drinkel.ow.org)
1717 */
1718 if ((ptype->dev == dev || !ptype->dev) &&
1719 (!skb_loop_sk(ptype, skb))) {
1720 if (pt_prev) {
1721 deliver_skb(skb2, pt_prev, skb->dev);
1722 pt_prev = ptype;
1723 continue;
1724 }
1725
1726 skb2 = skb_clone(skb, GFP_ATOMIC);
1727 if (!skb2)
1728 break;
1729
1730 net_timestamp_set(skb2);
1731
1732 /* skb->nh should be correctly
1733 set by sender, so that the second statement is
1734 just protection against buggy protocols.
1735 */
1736 skb_reset_mac_header(skb2);
1737
1738 if (skb_network_header(skb2) < skb2->data ||
1739 skb2->network_header > skb2->tail) {
1740 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1741 ntohs(skb2->protocol),
1742 dev->name);
1743 skb_reset_network_header(skb2);
1744 }
1745
1746 skb2->transport_header = skb2->network_header;
1747 skb2->pkt_type = PACKET_OUTGOING;
1748 pt_prev = ptype;
1749 }
1750 }
1751 if (pt_prev)
1752 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1753 rcu_read_unlock();
1754 }
1755
1756 /**
1757 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1758 * @dev: Network device
1759 * @txq: number of queues available
1760 *
1761 * If real_num_tx_queues is changed the tc mappings may no longer be
1762 * valid. To resolve this verify the tc mapping remains valid and if
1763 * not NULL the mapping. With no priorities mapping to this
1764 * offset/count pair it will no longer be used. In the worst case TC0
1765 * is invalid nothing can be done so disable priority mappings. If is
1766 * expected that drivers will fix this mapping if they can before
1767 * calling netif_set_real_num_tx_queues.
1768 */
netif_setup_tc(struct net_device * dev,unsigned int txq)1769 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1770 {
1771 int i;
1772 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1773
1774 /* If TC0 is invalidated disable TC mapping */
1775 if (tc->offset + tc->count > txq) {
1776 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1777 dev->num_tc = 0;
1778 return;
1779 }
1780
1781 /* Invalidated prio to tc mappings set to TC0 */
1782 for (i = 1; i < TC_BITMASK + 1; i++) {
1783 int q = netdev_get_prio_tc_map(dev, i);
1784
1785 tc = &dev->tc_to_txq[q];
1786 if (tc->offset + tc->count > txq) {
1787 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1788 i, q);
1789 netdev_set_prio_tc_map(dev, i, 0);
1790 }
1791 }
1792 }
1793
1794 #ifdef CONFIG_XPS
1795 static DEFINE_MUTEX(xps_map_mutex);
1796 #define xmap_dereference(P) \
1797 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1798
remove_xps_queue(struct xps_dev_maps * dev_maps,int cpu,u16 index)1799 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1800 int cpu, u16 index)
1801 {
1802 struct xps_map *map = NULL;
1803 int pos;
1804
1805 if (dev_maps)
1806 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1807
1808 for (pos = 0; map && pos < map->len; pos++) {
1809 if (map->queues[pos] == index) {
1810 if (map->len > 1) {
1811 map->queues[pos] = map->queues[--map->len];
1812 } else {
1813 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1814 kfree_rcu(map, rcu);
1815 map = NULL;
1816 }
1817 break;
1818 }
1819 }
1820
1821 return map;
1822 }
1823
netif_reset_xps_queues_gt(struct net_device * dev,u16 index)1824 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1825 {
1826 struct xps_dev_maps *dev_maps;
1827 int cpu, i;
1828 bool active = false;
1829
1830 mutex_lock(&xps_map_mutex);
1831 dev_maps = xmap_dereference(dev->xps_maps);
1832
1833 if (!dev_maps)
1834 goto out_no_maps;
1835
1836 for_each_possible_cpu(cpu) {
1837 for (i = index; i < dev->num_tx_queues; i++) {
1838 if (!remove_xps_queue(dev_maps, cpu, i))
1839 break;
1840 }
1841 if (i == dev->num_tx_queues)
1842 active = true;
1843 }
1844
1845 if (!active) {
1846 RCU_INIT_POINTER(dev->xps_maps, NULL);
1847 kfree_rcu(dev_maps, rcu);
1848 }
1849
1850 for (i = index; i < dev->num_tx_queues; i++)
1851 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1852 NUMA_NO_NODE);
1853
1854 out_no_maps:
1855 mutex_unlock(&xps_map_mutex);
1856 }
1857
expand_xps_map(struct xps_map * map,int cpu,u16 index)1858 static struct xps_map *expand_xps_map(struct xps_map *map,
1859 int cpu, u16 index)
1860 {
1861 struct xps_map *new_map;
1862 int alloc_len = XPS_MIN_MAP_ALLOC;
1863 int i, pos;
1864
1865 for (pos = 0; map && pos < map->len; pos++) {
1866 if (map->queues[pos] != index)
1867 continue;
1868 return map;
1869 }
1870
1871 /* Need to add queue to this CPU's existing map */
1872 if (map) {
1873 if (pos < map->alloc_len)
1874 return map;
1875
1876 alloc_len = map->alloc_len * 2;
1877 }
1878
1879 /* Need to allocate new map to store queue on this CPU's map */
1880 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1881 cpu_to_node(cpu));
1882 if (!new_map)
1883 return NULL;
1884
1885 for (i = 0; i < pos; i++)
1886 new_map->queues[i] = map->queues[i];
1887 new_map->alloc_len = alloc_len;
1888 new_map->len = pos;
1889
1890 return new_map;
1891 }
1892
netif_set_xps_queue(struct net_device * dev,struct cpumask * mask,u16 index)1893 int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)
1894 {
1895 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
1896 struct xps_map *map, *new_map;
1897 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
1898 int cpu, numa_node_id = -2;
1899 bool active = false;
1900
1901 mutex_lock(&xps_map_mutex);
1902
1903 dev_maps = xmap_dereference(dev->xps_maps);
1904
1905 /* allocate memory for queue storage */
1906 for_each_online_cpu(cpu) {
1907 if (!cpumask_test_cpu(cpu, mask))
1908 continue;
1909
1910 if (!new_dev_maps)
1911 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
1912 if (!new_dev_maps) {
1913 mutex_unlock(&xps_map_mutex);
1914 return -ENOMEM;
1915 }
1916
1917 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
1918 NULL;
1919
1920 map = expand_xps_map(map, cpu, index);
1921 if (!map)
1922 goto error;
1923
1924 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1925 }
1926
1927 if (!new_dev_maps)
1928 goto out_no_new_maps;
1929
1930 for_each_possible_cpu(cpu) {
1931 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
1932 /* add queue to CPU maps */
1933 int pos = 0;
1934
1935 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1936 while ((pos < map->len) && (map->queues[pos] != index))
1937 pos++;
1938
1939 if (pos == map->len)
1940 map->queues[map->len++] = index;
1941 #ifdef CONFIG_NUMA
1942 if (numa_node_id == -2)
1943 numa_node_id = cpu_to_node(cpu);
1944 else if (numa_node_id != cpu_to_node(cpu))
1945 numa_node_id = -1;
1946 #endif
1947 } else if (dev_maps) {
1948 /* fill in the new device map from the old device map */
1949 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1950 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1951 }
1952
1953 }
1954
1955 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
1956
1957 /* Cleanup old maps */
1958 if (dev_maps) {
1959 for_each_possible_cpu(cpu) {
1960 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1961 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1962 if (map && map != new_map)
1963 kfree_rcu(map, rcu);
1964 }
1965
1966 kfree_rcu(dev_maps, rcu);
1967 }
1968
1969 dev_maps = new_dev_maps;
1970 active = true;
1971
1972 out_no_new_maps:
1973 /* update Tx queue numa node */
1974 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
1975 (numa_node_id >= 0) ? numa_node_id :
1976 NUMA_NO_NODE);
1977
1978 if (!dev_maps)
1979 goto out_no_maps;
1980
1981 /* removes queue from unused CPUs */
1982 for_each_possible_cpu(cpu) {
1983 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
1984 continue;
1985
1986 if (remove_xps_queue(dev_maps, cpu, index))
1987 active = true;
1988 }
1989
1990 /* free map if not active */
1991 if (!active) {
1992 RCU_INIT_POINTER(dev->xps_maps, NULL);
1993 kfree_rcu(dev_maps, rcu);
1994 }
1995
1996 out_no_maps:
1997 mutex_unlock(&xps_map_mutex);
1998
1999 return 0;
2000 error:
2001 /* remove any maps that we added */
2002 for_each_possible_cpu(cpu) {
2003 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2004 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2005 NULL;
2006 if (new_map && new_map != map)
2007 kfree(new_map);
2008 }
2009
2010 mutex_unlock(&xps_map_mutex);
2011
2012 kfree(new_dev_maps);
2013 return -ENOMEM;
2014 }
2015 EXPORT_SYMBOL(netif_set_xps_queue);
2016
2017 #endif
2018 /*
2019 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2020 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2021 */
netif_set_real_num_tx_queues(struct net_device * dev,unsigned int txq)2022 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2023 {
2024 int rc;
2025
2026 if (txq < 1 || txq > dev->num_tx_queues)
2027 return -EINVAL;
2028
2029 if (dev->reg_state == NETREG_REGISTERED ||
2030 dev->reg_state == NETREG_UNREGISTERING) {
2031 ASSERT_RTNL();
2032
2033 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2034 txq);
2035 if (rc)
2036 return rc;
2037
2038 if (dev->num_tc)
2039 netif_setup_tc(dev, txq);
2040
2041 if (txq < dev->real_num_tx_queues) {
2042 qdisc_reset_all_tx_gt(dev, txq);
2043 #ifdef CONFIG_XPS
2044 netif_reset_xps_queues_gt(dev, txq);
2045 #endif
2046 }
2047 }
2048
2049 dev->real_num_tx_queues = txq;
2050 return 0;
2051 }
2052 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2053
2054 #ifdef CONFIG_RPS
2055 /**
2056 * netif_set_real_num_rx_queues - set actual number of RX queues used
2057 * @dev: Network device
2058 * @rxq: Actual number of RX queues
2059 *
2060 * This must be called either with the rtnl_lock held or before
2061 * registration of the net device. Returns 0 on success, or a
2062 * negative error code. If called before registration, it always
2063 * succeeds.
2064 */
netif_set_real_num_rx_queues(struct net_device * dev,unsigned int rxq)2065 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2066 {
2067 int rc;
2068
2069 if (rxq < 1 || rxq > dev->num_rx_queues)
2070 return -EINVAL;
2071
2072 if (dev->reg_state == NETREG_REGISTERED) {
2073 ASSERT_RTNL();
2074
2075 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2076 rxq);
2077 if (rc)
2078 return rc;
2079 }
2080
2081 dev->real_num_rx_queues = rxq;
2082 return 0;
2083 }
2084 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2085 #endif
2086
2087 /**
2088 * netif_get_num_default_rss_queues - default number of RSS queues
2089 *
2090 * This routine should set an upper limit on the number of RSS queues
2091 * used by default by multiqueue devices.
2092 */
netif_get_num_default_rss_queues(void)2093 int netif_get_num_default_rss_queues(void)
2094 {
2095 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2096 }
2097 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2098
__netif_reschedule(struct Qdisc * q)2099 static inline void __netif_reschedule(struct Qdisc *q)
2100 {
2101 struct softnet_data *sd;
2102 unsigned long flags;
2103
2104 local_irq_save(flags);
2105 sd = &__get_cpu_var(softnet_data);
2106 q->next_sched = NULL;
2107 *sd->output_queue_tailp = q;
2108 sd->output_queue_tailp = &q->next_sched;
2109 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2110 local_irq_restore(flags);
2111 }
2112
__netif_schedule(struct Qdisc * q)2113 void __netif_schedule(struct Qdisc *q)
2114 {
2115 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2116 __netif_reschedule(q);
2117 }
2118 EXPORT_SYMBOL(__netif_schedule);
2119
dev_kfree_skb_irq(struct sk_buff * skb)2120 void dev_kfree_skb_irq(struct sk_buff *skb)
2121 {
2122 if (atomic_dec_and_test(&skb->users)) {
2123 struct softnet_data *sd;
2124 unsigned long flags;
2125
2126 local_irq_save(flags);
2127 sd = &__get_cpu_var(softnet_data);
2128 skb->next = sd->completion_queue;
2129 sd->completion_queue = skb;
2130 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2131 local_irq_restore(flags);
2132 }
2133 }
2134 EXPORT_SYMBOL(dev_kfree_skb_irq);
2135
dev_kfree_skb_any(struct sk_buff * skb)2136 void dev_kfree_skb_any(struct sk_buff *skb)
2137 {
2138 if (in_irq() || irqs_disabled())
2139 dev_kfree_skb_irq(skb);
2140 else
2141 dev_kfree_skb(skb);
2142 }
2143 EXPORT_SYMBOL(dev_kfree_skb_any);
2144
2145
2146 /**
2147 * netif_device_detach - mark device as removed
2148 * @dev: network device
2149 *
2150 * Mark device as removed from system and therefore no longer available.
2151 */
netif_device_detach(struct net_device * dev)2152 void netif_device_detach(struct net_device *dev)
2153 {
2154 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2155 netif_running(dev)) {
2156 netif_tx_stop_all_queues(dev);
2157 }
2158 }
2159 EXPORT_SYMBOL(netif_device_detach);
2160
2161 /**
2162 * netif_device_attach - mark device as attached
2163 * @dev: network device
2164 *
2165 * Mark device as attached from system and restart if needed.
2166 */
netif_device_attach(struct net_device * dev)2167 void netif_device_attach(struct net_device *dev)
2168 {
2169 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2170 netif_running(dev)) {
2171 netif_tx_wake_all_queues(dev);
2172 __netdev_watchdog_up(dev);
2173 }
2174 }
2175 EXPORT_SYMBOL(netif_device_attach);
2176
skb_warn_bad_offload(const struct sk_buff * skb)2177 static void skb_warn_bad_offload(const struct sk_buff *skb)
2178 {
2179 static const netdev_features_t null_features = 0;
2180 struct net_device *dev = skb->dev;
2181 const char *driver = "";
2182
2183 if (!net_ratelimit())
2184 return;
2185
2186 if (dev && dev->dev.parent)
2187 driver = dev_driver_string(dev->dev.parent);
2188
2189 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2190 "gso_type=%d ip_summed=%d\n",
2191 driver, dev ? &dev->features : &null_features,
2192 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2193 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2194 skb_shinfo(skb)->gso_type, skb->ip_summed);
2195 }
2196
2197 /*
2198 * Invalidate hardware checksum when packet is to be mangled, and
2199 * complete checksum manually on outgoing path.
2200 */
skb_checksum_help(struct sk_buff * skb)2201 int skb_checksum_help(struct sk_buff *skb)
2202 {
2203 __wsum csum;
2204 int ret = 0, offset;
2205
2206 if (skb->ip_summed == CHECKSUM_COMPLETE)
2207 goto out_set_summed;
2208
2209 if (unlikely(skb_shinfo(skb)->gso_size)) {
2210 skb_warn_bad_offload(skb);
2211 return -EINVAL;
2212 }
2213
2214 /* Before computing a checksum, we should make sure no frag could
2215 * be modified by an external entity : checksum could be wrong.
2216 */
2217 if (skb_has_shared_frag(skb)) {
2218 ret = __skb_linearize(skb);
2219 if (ret)
2220 goto out;
2221 }
2222
2223 offset = skb_checksum_start_offset(skb);
2224 BUG_ON(offset >= skb_headlen(skb));
2225 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2226
2227 offset += skb->csum_offset;
2228 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2229
2230 if (skb_cloned(skb) &&
2231 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2232 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2233 if (ret)
2234 goto out;
2235 }
2236
2237 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2238 out_set_summed:
2239 skb->ip_summed = CHECKSUM_NONE;
2240 out:
2241 return ret;
2242 }
2243 EXPORT_SYMBOL(skb_checksum_help);
2244
skb_network_protocol(struct sk_buff * skb)2245 __be16 skb_network_protocol(struct sk_buff *skb)
2246 {
2247 __be16 type = skb->protocol;
2248 int vlan_depth = ETH_HLEN;
2249
2250 /* Tunnel gso handlers can set protocol to ethernet. */
2251 if (type == htons(ETH_P_TEB)) {
2252 struct ethhdr *eth;
2253
2254 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2255 return 0;
2256
2257 eth = (struct ethhdr *)skb_mac_header(skb);
2258 type = eth->h_proto;
2259 }
2260
2261 while (type == htons(ETH_P_8021Q) || type == htons(ETH_P_8021AD)) {
2262 struct vlan_hdr *vh;
2263
2264 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
2265 return 0;
2266
2267 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2268 type = vh->h_vlan_encapsulated_proto;
2269 vlan_depth += VLAN_HLEN;
2270 }
2271
2272 return type;
2273 }
2274
2275 /**
2276 * skb_mac_gso_segment - mac layer segmentation handler.
2277 * @skb: buffer to segment
2278 * @features: features for the output path (see dev->features)
2279 */
skb_mac_gso_segment(struct sk_buff * skb,netdev_features_t features)2280 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2281 netdev_features_t features)
2282 {
2283 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2284 struct packet_offload *ptype;
2285 __be16 type = skb_network_protocol(skb);
2286
2287 if (unlikely(!type))
2288 return ERR_PTR(-EINVAL);
2289
2290 __skb_pull(skb, skb->mac_len);
2291
2292 rcu_read_lock();
2293 list_for_each_entry_rcu(ptype, &offload_base, list) {
2294 if (ptype->type == type && ptype->callbacks.gso_segment) {
2295 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
2296 int err;
2297
2298 err = ptype->callbacks.gso_send_check(skb);
2299 segs = ERR_PTR(err);
2300 if (err || skb_gso_ok(skb, features))
2301 break;
2302 __skb_push(skb, (skb->data -
2303 skb_network_header(skb)));
2304 }
2305 segs = ptype->callbacks.gso_segment(skb, features);
2306 break;
2307 }
2308 }
2309 rcu_read_unlock();
2310
2311 __skb_push(skb, skb->data - skb_mac_header(skb));
2312
2313 return segs;
2314 }
2315 EXPORT_SYMBOL(skb_mac_gso_segment);
2316
2317
2318 /* openvswitch calls this on rx path, so we need a different check.
2319 */
skb_needs_check(struct sk_buff * skb,bool tx_path)2320 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2321 {
2322 if (tx_path)
2323 return skb->ip_summed != CHECKSUM_PARTIAL;
2324 else
2325 return skb->ip_summed == CHECKSUM_NONE;
2326 }
2327
2328 /**
2329 * __skb_gso_segment - Perform segmentation on skb.
2330 * @skb: buffer to segment
2331 * @features: features for the output path (see dev->features)
2332 * @tx_path: whether it is called in TX path
2333 *
2334 * This function segments the given skb and returns a list of segments.
2335 *
2336 * It may return NULL if the skb requires no segmentation. This is
2337 * only possible when GSO is used for verifying header integrity.
2338 */
__skb_gso_segment(struct sk_buff * skb,netdev_features_t features,bool tx_path)2339 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2340 netdev_features_t features, bool tx_path)
2341 {
2342 if (unlikely(skb_needs_check(skb, tx_path))) {
2343 int err;
2344
2345 skb_warn_bad_offload(skb);
2346
2347 if (skb_header_cloned(skb) &&
2348 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2349 return ERR_PTR(err);
2350 }
2351
2352 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2353 skb_reset_mac_header(skb);
2354 skb_reset_mac_len(skb);
2355
2356 return skb_mac_gso_segment(skb, features);
2357 }
2358 EXPORT_SYMBOL(__skb_gso_segment);
2359
2360 /* Take action when hardware reception checksum errors are detected. */
2361 #ifdef CONFIG_BUG
netdev_rx_csum_fault(struct net_device * dev)2362 void netdev_rx_csum_fault(struct net_device *dev)
2363 {
2364 if (net_ratelimit()) {
2365 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2366 dump_stack();
2367 }
2368 }
2369 EXPORT_SYMBOL(netdev_rx_csum_fault);
2370 #endif
2371
2372 /* Actually, we should eliminate this check as soon as we know, that:
2373 * 1. IOMMU is present and allows to map all the memory.
2374 * 2. No high memory really exists on this machine.
2375 */
2376
illegal_highdma(struct net_device * dev,struct sk_buff * skb)2377 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2378 {
2379 #ifdef CONFIG_HIGHMEM
2380 int i;
2381 if (!(dev->features & NETIF_F_HIGHDMA)) {
2382 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2383 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2384 if (PageHighMem(skb_frag_page(frag)))
2385 return 1;
2386 }
2387 }
2388
2389 if (PCI_DMA_BUS_IS_PHYS) {
2390 struct device *pdev = dev->dev.parent;
2391
2392 if (!pdev)
2393 return 0;
2394 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2395 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2396 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2397 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2398 return 1;
2399 }
2400 }
2401 #endif
2402 return 0;
2403 }
2404
2405 struct dev_gso_cb {
2406 void (*destructor)(struct sk_buff *skb);
2407 };
2408
2409 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2410
dev_gso_skb_destructor(struct sk_buff * skb)2411 static void dev_gso_skb_destructor(struct sk_buff *skb)
2412 {
2413 struct dev_gso_cb *cb;
2414
2415 do {
2416 struct sk_buff *nskb = skb->next;
2417
2418 skb->next = nskb->next;
2419 nskb->next = NULL;
2420 kfree_skb(nskb);
2421 } while (skb->next);
2422
2423 cb = DEV_GSO_CB(skb);
2424 if (cb->destructor)
2425 cb->destructor(skb);
2426 }
2427
2428 /**
2429 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2430 * @skb: buffer to segment
2431 * @features: device features as applicable to this skb
2432 *
2433 * This function segments the given skb and stores the list of segments
2434 * in skb->next.
2435 */
dev_gso_segment(struct sk_buff * skb,netdev_features_t features)2436 static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
2437 {
2438 struct sk_buff *segs;
2439
2440 segs = skb_gso_segment(skb, features);
2441
2442 /* Verifying header integrity only. */
2443 if (!segs)
2444 return 0;
2445
2446 if (IS_ERR(segs))
2447 return PTR_ERR(segs);
2448
2449 skb->next = segs;
2450 DEV_GSO_CB(skb)->destructor = skb->destructor;
2451 skb->destructor = dev_gso_skb_destructor;
2452
2453 return 0;
2454 }
2455
harmonize_features(struct sk_buff * skb,__be16 protocol,netdev_features_t features)2456 static netdev_features_t harmonize_features(struct sk_buff *skb,
2457 __be16 protocol, netdev_features_t features)
2458 {
2459 if (skb->ip_summed != CHECKSUM_NONE &&
2460 !can_checksum_protocol(features, protocol)) {
2461 features &= ~NETIF_F_ALL_CSUM;
2462 } else if (illegal_highdma(skb->dev, skb)) {
2463 features &= ~NETIF_F_SG;
2464 }
2465
2466 return features;
2467 }
2468
netif_skb_features(struct sk_buff * skb)2469 netdev_features_t netif_skb_features(struct sk_buff *skb)
2470 {
2471 __be16 protocol = skb->protocol;
2472 netdev_features_t features = skb->dev->features;
2473
2474 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2475 features &= ~NETIF_F_GSO_MASK;
2476
2477 if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD)) {
2478 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2479 protocol = veh->h_vlan_encapsulated_proto;
2480 } else if (!vlan_tx_tag_present(skb)) {
2481 return harmonize_features(skb, protocol, features);
2482 }
2483
2484 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_CTAG_TX |
2485 NETIF_F_HW_VLAN_STAG_TX);
2486
2487 if (protocol != htons(ETH_P_8021Q) && protocol != htons(ETH_P_8021AD)) {
2488 return harmonize_features(skb, protocol, features);
2489 } else {
2490 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
2491 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2492 NETIF_F_HW_VLAN_STAG_TX;
2493 return harmonize_features(skb, protocol, features);
2494 }
2495 }
2496 EXPORT_SYMBOL(netif_skb_features);
2497
2498 /*
2499 * Returns true if either:
2500 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2501 * 2. skb is fragmented and the device does not support SG.
2502 */
skb_needs_linearize(struct sk_buff * skb,netdev_features_t features)2503 static inline int skb_needs_linearize(struct sk_buff *skb,
2504 netdev_features_t features)
2505 {
2506 return skb_is_nonlinear(skb) &&
2507 ((skb_has_frag_list(skb) &&
2508 !(features & NETIF_F_FRAGLIST)) ||
2509 (skb_shinfo(skb)->nr_frags &&
2510 !(features & NETIF_F_SG)));
2511 }
2512
dev_hard_start_xmit(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq)2513 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2514 struct netdev_queue *txq)
2515 {
2516 const struct net_device_ops *ops = dev->netdev_ops;
2517 int rc = NETDEV_TX_OK;
2518 unsigned int skb_len;
2519
2520 if (likely(!skb->next)) {
2521 netdev_features_t features;
2522
2523 /*
2524 * If device doesn't need skb->dst, release it right now while
2525 * its hot in this cpu cache
2526 */
2527 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2528 skb_dst_drop(skb);
2529
2530 features = netif_skb_features(skb);
2531
2532 if (vlan_tx_tag_present(skb) &&
2533 !vlan_hw_offload_capable(features, skb->vlan_proto)) {
2534 skb = __vlan_put_tag(skb, skb->vlan_proto,
2535 vlan_tx_tag_get(skb));
2536 if (unlikely(!skb))
2537 goto out;
2538
2539 skb->vlan_tci = 0;
2540 }
2541
2542 /* If encapsulation offload request, verify we are testing
2543 * hardware encapsulation features instead of standard
2544 * features for the netdev
2545 */
2546 if (skb->encapsulation)
2547 features &= dev->hw_enc_features;
2548
2549 if (netif_needs_gso(skb, features)) {
2550 if (unlikely(dev_gso_segment(skb, features)))
2551 goto out_kfree_skb;
2552 if (skb->next)
2553 goto gso;
2554 } else {
2555 if (skb_needs_linearize(skb, features) &&
2556 __skb_linearize(skb))
2557 goto out_kfree_skb;
2558
2559 /* If packet is not checksummed and device does not
2560 * support checksumming for this protocol, complete
2561 * checksumming here.
2562 */
2563 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2564 if (skb->encapsulation)
2565 skb_set_inner_transport_header(skb,
2566 skb_checksum_start_offset(skb));
2567 else
2568 skb_set_transport_header(skb,
2569 skb_checksum_start_offset(skb));
2570 if (!(features & NETIF_F_ALL_CSUM) &&
2571 skb_checksum_help(skb))
2572 goto out_kfree_skb;
2573 }
2574 }
2575
2576 if (!list_empty(&ptype_all))
2577 dev_queue_xmit_nit(skb, dev);
2578
2579 skb_len = skb->len;
2580 rc = ops->ndo_start_xmit(skb, dev);
2581 trace_net_dev_xmit(skb, rc, dev, skb_len);
2582 if (rc == NETDEV_TX_OK)
2583 txq_trans_update(txq);
2584 return rc;
2585 }
2586
2587 gso:
2588 do {
2589 struct sk_buff *nskb = skb->next;
2590
2591 skb->next = nskb->next;
2592 nskb->next = NULL;
2593
2594 if (!list_empty(&ptype_all))
2595 dev_queue_xmit_nit(nskb, dev);
2596
2597 skb_len = nskb->len;
2598 rc = ops->ndo_start_xmit(nskb, dev);
2599 trace_net_dev_xmit(nskb, rc, dev, skb_len);
2600 if (unlikely(rc != NETDEV_TX_OK)) {
2601 if (rc & ~NETDEV_TX_MASK)
2602 goto out_kfree_gso_skb;
2603 nskb->next = skb->next;
2604 skb->next = nskb;
2605 return rc;
2606 }
2607 txq_trans_update(txq);
2608 if (unlikely(netif_xmit_stopped(txq) && skb->next))
2609 return NETDEV_TX_BUSY;
2610 } while (skb->next);
2611
2612 out_kfree_gso_skb:
2613 if (likely(skb->next == NULL)) {
2614 skb->destructor = DEV_GSO_CB(skb)->destructor;
2615 consume_skb(skb);
2616 return rc;
2617 }
2618 out_kfree_skb:
2619 kfree_skb(skb);
2620 out:
2621 return rc;
2622 }
2623
qdisc_pkt_len_init(struct sk_buff * skb)2624 static void qdisc_pkt_len_init(struct sk_buff *skb)
2625 {
2626 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2627
2628 qdisc_skb_cb(skb)->pkt_len = skb->len;
2629
2630 /* To get more precise estimation of bytes sent on wire,
2631 * we add to pkt_len the headers size of all segments
2632 */
2633 if (shinfo->gso_size) {
2634 unsigned int hdr_len;
2635 u16 gso_segs = shinfo->gso_segs;
2636
2637 /* mac layer + network layer */
2638 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2639
2640 /* + transport layer */
2641 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2642 hdr_len += tcp_hdrlen(skb);
2643 else
2644 hdr_len += sizeof(struct udphdr);
2645
2646 if (shinfo->gso_type & SKB_GSO_DODGY)
2647 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2648 shinfo->gso_size);
2649
2650 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
2651 }
2652 }
2653
__dev_xmit_skb(struct sk_buff * skb,struct Qdisc * q,struct net_device * dev,struct netdev_queue * txq)2654 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2655 struct net_device *dev,
2656 struct netdev_queue *txq)
2657 {
2658 spinlock_t *root_lock = qdisc_lock(q);
2659 bool contended;
2660 int rc;
2661
2662 qdisc_pkt_len_init(skb);
2663 qdisc_calculate_pkt_len(skb, q);
2664 /*
2665 * Heuristic to force contended enqueues to serialize on a
2666 * separate lock before trying to get qdisc main lock.
2667 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2668 * and dequeue packets faster.
2669 */
2670 contended = qdisc_is_running(q);
2671 if (unlikely(contended))
2672 spin_lock(&q->busylock);
2673
2674 spin_lock(root_lock);
2675 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2676 kfree_skb(skb);
2677 rc = NET_XMIT_DROP;
2678 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2679 qdisc_run_begin(q)) {
2680 /*
2681 * This is a work-conserving queue; there are no old skbs
2682 * waiting to be sent out; and the qdisc is not running -
2683 * xmit the skb directly.
2684 */
2685 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2686 skb_dst_force(skb);
2687
2688 qdisc_bstats_update(q, skb);
2689
2690 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2691 if (unlikely(contended)) {
2692 spin_unlock(&q->busylock);
2693 contended = false;
2694 }
2695 __qdisc_run(q);
2696 } else
2697 qdisc_run_end(q);
2698
2699 rc = NET_XMIT_SUCCESS;
2700 } else {
2701 skb_dst_force(skb);
2702 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2703 if (qdisc_run_begin(q)) {
2704 if (unlikely(contended)) {
2705 spin_unlock(&q->busylock);
2706 contended = false;
2707 }
2708 __qdisc_run(q);
2709 }
2710 }
2711 spin_unlock(root_lock);
2712 if (unlikely(contended))
2713 spin_unlock(&q->busylock);
2714 return rc;
2715 }
2716
2717 #if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
skb_update_prio(struct sk_buff * skb)2718 static void skb_update_prio(struct sk_buff *skb)
2719 {
2720 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
2721
2722 if (!skb->priority && skb->sk && map) {
2723 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2724
2725 if (prioidx < map->priomap_len)
2726 skb->priority = map->priomap[prioidx];
2727 }
2728 }
2729 #else
2730 #define skb_update_prio(skb)
2731 #endif
2732
2733 static DEFINE_PER_CPU(int, xmit_recursion);
2734 #define RECURSION_LIMIT 10
2735
2736 /**
2737 * dev_loopback_xmit - loop back @skb
2738 * @skb: buffer to transmit
2739 */
dev_loopback_xmit(struct sk_buff * skb)2740 int dev_loopback_xmit(struct sk_buff *skb)
2741 {
2742 skb_reset_mac_header(skb);
2743 __skb_pull(skb, skb_network_offset(skb));
2744 skb->pkt_type = PACKET_LOOPBACK;
2745 skb->ip_summed = CHECKSUM_UNNECESSARY;
2746 WARN_ON(!skb_dst(skb));
2747 skb_dst_force(skb);
2748 netif_rx_ni(skb);
2749 return 0;
2750 }
2751 EXPORT_SYMBOL(dev_loopback_xmit);
2752
2753 /**
2754 * dev_queue_xmit - transmit a buffer
2755 * @skb: buffer to transmit
2756 *
2757 * Queue a buffer for transmission to a network device. The caller must
2758 * have set the device and priority and built the buffer before calling
2759 * this function. The function can be called from an interrupt.
2760 *
2761 * A negative errno code is returned on a failure. A success does not
2762 * guarantee the frame will be transmitted as it may be dropped due
2763 * to congestion or traffic shaping.
2764 *
2765 * -----------------------------------------------------------------------------------
2766 * I notice this method can also return errors from the queue disciplines,
2767 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2768 * be positive.
2769 *
2770 * Regardless of the return value, the skb is consumed, so it is currently
2771 * difficult to retry a send to this method. (You can bump the ref count
2772 * before sending to hold a reference for retry if you are careful.)
2773 *
2774 * When calling this method, interrupts MUST be enabled. This is because
2775 * the BH enable code must have IRQs enabled so that it will not deadlock.
2776 * --BLG
2777 */
dev_queue_xmit(struct sk_buff * skb)2778 int dev_queue_xmit(struct sk_buff *skb)
2779 {
2780 struct net_device *dev = skb->dev;
2781 struct netdev_queue *txq;
2782 struct Qdisc *q;
2783 int rc = -ENOMEM;
2784
2785 skb_reset_mac_header(skb);
2786
2787 /* Disable soft irqs for various locks below. Also
2788 * stops preemption for RCU.
2789 */
2790 rcu_read_lock_bh();
2791
2792 skb_update_prio(skb);
2793
2794 txq = netdev_pick_tx(dev, skb);
2795 q = rcu_dereference_bh(txq->qdisc);
2796
2797 #ifdef CONFIG_NET_CLS_ACT
2798 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2799 #endif
2800 trace_net_dev_queue(skb);
2801 if (q->enqueue) {
2802 rc = __dev_xmit_skb(skb, q, dev, txq);
2803 goto out;
2804 }
2805
2806 /* The device has no queue. Common case for software devices:
2807 loopback, all the sorts of tunnels...
2808
2809 Really, it is unlikely that netif_tx_lock protection is necessary
2810 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2811 counters.)
2812 However, it is possible, that they rely on protection
2813 made by us here.
2814
2815 Check this and shot the lock. It is not prone from deadlocks.
2816 Either shot noqueue qdisc, it is even simpler 8)
2817 */
2818 if (dev->flags & IFF_UP) {
2819 int cpu = smp_processor_id(); /* ok because BHs are off */
2820
2821 if (txq->xmit_lock_owner != cpu) {
2822
2823 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2824 goto recursion_alert;
2825
2826 HARD_TX_LOCK(dev, txq, cpu);
2827
2828 if (!netif_xmit_stopped(txq)) {
2829 __this_cpu_inc(xmit_recursion);
2830 rc = dev_hard_start_xmit(skb, dev, txq);
2831 __this_cpu_dec(xmit_recursion);
2832 if (dev_xmit_complete(rc)) {
2833 HARD_TX_UNLOCK(dev, txq);
2834 goto out;
2835 }
2836 }
2837 HARD_TX_UNLOCK(dev, txq);
2838 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2839 dev->name);
2840 } else {
2841 /* Recursion is detected! It is possible,
2842 * unfortunately
2843 */
2844 recursion_alert:
2845 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2846 dev->name);
2847 }
2848 }
2849
2850 rc = -ENETDOWN;
2851 rcu_read_unlock_bh();
2852
2853 kfree_skb(skb);
2854 return rc;
2855 out:
2856 rcu_read_unlock_bh();
2857 return rc;
2858 }
2859 EXPORT_SYMBOL(dev_queue_xmit);
2860
2861
2862 /*=======================================================================
2863 Receiver routines
2864 =======================================================================*/
2865
2866 int netdev_max_backlog __read_mostly = 1000;
2867 EXPORT_SYMBOL(netdev_max_backlog);
2868
2869 int netdev_tstamp_prequeue __read_mostly = 1;
2870 int netdev_budget __read_mostly = 300;
2871 int weight_p __read_mostly = 64; /* old backlog weight */
2872
2873 /* Called with irq disabled */
____napi_schedule(struct softnet_data * sd,struct napi_struct * napi)2874 static inline void ____napi_schedule(struct softnet_data *sd,
2875 struct napi_struct *napi)
2876 {
2877 list_add_tail(&napi->poll_list, &sd->poll_list);
2878 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2879 }
2880
2881 #ifdef CONFIG_RPS
2882
2883 /* One global table that all flow-based protocols share. */
2884 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2885 EXPORT_SYMBOL(rps_sock_flow_table);
2886
2887 struct static_key rps_needed __read_mostly;
2888
2889 static struct rps_dev_flow *
set_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow * rflow,u16 next_cpu)2890 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2891 struct rps_dev_flow *rflow, u16 next_cpu)
2892 {
2893 if (next_cpu != RPS_NO_CPU) {
2894 #ifdef CONFIG_RFS_ACCEL
2895 struct netdev_rx_queue *rxqueue;
2896 struct rps_dev_flow_table *flow_table;
2897 struct rps_dev_flow *old_rflow;
2898 u32 flow_id;
2899 u16 rxq_index;
2900 int rc;
2901
2902 /* Should we steer this flow to a different hardware queue? */
2903 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2904 !(dev->features & NETIF_F_NTUPLE))
2905 goto out;
2906 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2907 if (rxq_index == skb_get_rx_queue(skb))
2908 goto out;
2909
2910 rxqueue = dev->_rx + rxq_index;
2911 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2912 if (!flow_table)
2913 goto out;
2914 flow_id = skb->rxhash & flow_table->mask;
2915 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2916 rxq_index, flow_id);
2917 if (rc < 0)
2918 goto out;
2919 old_rflow = rflow;
2920 rflow = &flow_table->flows[flow_id];
2921 rflow->filter = rc;
2922 if (old_rflow->filter == rflow->filter)
2923 old_rflow->filter = RPS_NO_FILTER;
2924 out:
2925 #endif
2926 rflow->last_qtail =
2927 per_cpu(softnet_data, next_cpu).input_queue_head;
2928 }
2929
2930 rflow->cpu = next_cpu;
2931 return rflow;
2932 }
2933
2934 /*
2935 * get_rps_cpu is called from netif_receive_skb and returns the target
2936 * CPU from the RPS map of the receiving queue for a given skb.
2937 * rcu_read_lock must be held on entry.
2938 */
get_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow ** rflowp)2939 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2940 struct rps_dev_flow **rflowp)
2941 {
2942 struct netdev_rx_queue *rxqueue;
2943 struct rps_map *map;
2944 struct rps_dev_flow_table *flow_table;
2945 struct rps_sock_flow_table *sock_flow_table;
2946 int cpu = -1;
2947 u16 tcpu;
2948
2949 if (skb_rx_queue_recorded(skb)) {
2950 u16 index = skb_get_rx_queue(skb);
2951 if (unlikely(index >= dev->real_num_rx_queues)) {
2952 WARN_ONCE(dev->real_num_rx_queues > 1,
2953 "%s received packet on queue %u, but number "
2954 "of RX queues is %u\n",
2955 dev->name, index, dev->real_num_rx_queues);
2956 goto done;
2957 }
2958 rxqueue = dev->_rx + index;
2959 } else
2960 rxqueue = dev->_rx;
2961
2962 map = rcu_dereference(rxqueue->rps_map);
2963 if (map) {
2964 if (map->len == 1 &&
2965 !rcu_access_pointer(rxqueue->rps_flow_table)) {
2966 tcpu = map->cpus[0];
2967 if (cpu_online(tcpu))
2968 cpu = tcpu;
2969 goto done;
2970 }
2971 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
2972 goto done;
2973 }
2974
2975 skb_reset_network_header(skb);
2976 if (!skb_get_rxhash(skb))
2977 goto done;
2978
2979 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2980 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2981 if (flow_table && sock_flow_table) {
2982 u16 next_cpu;
2983 struct rps_dev_flow *rflow;
2984
2985 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2986 tcpu = rflow->cpu;
2987
2988 next_cpu = sock_flow_table->ents[skb->rxhash &
2989 sock_flow_table->mask];
2990
2991 /*
2992 * If the desired CPU (where last recvmsg was done) is
2993 * different from current CPU (one in the rx-queue flow
2994 * table entry), switch if one of the following holds:
2995 * - Current CPU is unset (equal to RPS_NO_CPU).
2996 * - Current CPU is offline.
2997 * - The current CPU's queue tail has advanced beyond the
2998 * last packet that was enqueued using this table entry.
2999 * This guarantees that all previous packets for the flow
3000 * have been dequeued, thus preserving in order delivery.
3001 */
3002 if (unlikely(tcpu != next_cpu) &&
3003 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
3004 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3005 rflow->last_qtail)) >= 0)) {
3006 tcpu = next_cpu;
3007 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3008 }
3009
3010 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
3011 *rflowp = rflow;
3012 cpu = tcpu;
3013 goto done;
3014 }
3015 }
3016
3017 if (map) {
3018 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
3019
3020 if (cpu_online(tcpu)) {
3021 cpu = tcpu;
3022 goto done;
3023 }
3024 }
3025
3026 done:
3027 return cpu;
3028 }
3029
3030 #ifdef CONFIG_RFS_ACCEL
3031
3032 /**
3033 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3034 * @dev: Device on which the filter was set
3035 * @rxq_index: RX queue index
3036 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3037 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3038 *
3039 * Drivers that implement ndo_rx_flow_steer() should periodically call
3040 * this function for each installed filter and remove the filters for
3041 * which it returns %true.
3042 */
rps_may_expire_flow(struct net_device * dev,u16 rxq_index,u32 flow_id,u16 filter_id)3043 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3044 u32 flow_id, u16 filter_id)
3045 {
3046 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3047 struct rps_dev_flow_table *flow_table;
3048 struct rps_dev_flow *rflow;
3049 bool expire = true;
3050 int cpu;
3051
3052 rcu_read_lock();
3053 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3054 if (flow_table && flow_id <= flow_table->mask) {
3055 rflow = &flow_table->flows[flow_id];
3056 cpu = ACCESS_ONCE(rflow->cpu);
3057 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
3058 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3059 rflow->last_qtail) <
3060 (int)(10 * flow_table->mask)))
3061 expire = false;
3062 }
3063 rcu_read_unlock();
3064 return expire;
3065 }
3066 EXPORT_SYMBOL(rps_may_expire_flow);
3067
3068 #endif /* CONFIG_RFS_ACCEL */
3069
3070 /* Called from hardirq (IPI) context */
rps_trigger_softirq(void * data)3071 static void rps_trigger_softirq(void *data)
3072 {
3073 struct softnet_data *sd = data;
3074
3075 ____napi_schedule(sd, &sd->backlog);
3076 sd->received_rps++;
3077 }
3078
3079 #endif /* CONFIG_RPS */
3080
3081 /*
3082 * Check if this softnet_data structure is another cpu one
3083 * If yes, queue it to our IPI list and return 1
3084 * If no, return 0
3085 */
rps_ipi_queued(struct softnet_data * sd)3086 static int rps_ipi_queued(struct softnet_data *sd)
3087 {
3088 #ifdef CONFIG_RPS
3089 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
3090
3091 if (sd != mysd) {
3092 sd->rps_ipi_next = mysd->rps_ipi_list;
3093 mysd->rps_ipi_list = sd;
3094
3095 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3096 return 1;
3097 }
3098 #endif /* CONFIG_RPS */
3099 return 0;
3100 }
3101
3102 /*
3103 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3104 * queue (may be a remote CPU queue).
3105 */
enqueue_to_backlog(struct sk_buff * skb,int cpu,unsigned int * qtail)3106 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3107 unsigned int *qtail)
3108 {
3109 struct softnet_data *sd;
3110 unsigned long flags;
3111
3112 sd = &per_cpu(softnet_data, cpu);
3113
3114 local_irq_save(flags);
3115
3116 rps_lock(sd);
3117 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
3118 if (skb_queue_len(&sd->input_pkt_queue)) {
3119 enqueue:
3120 __skb_queue_tail(&sd->input_pkt_queue, skb);
3121 input_queue_tail_incr_save(sd, qtail);
3122 rps_unlock(sd);
3123 local_irq_restore(flags);
3124 return NET_RX_SUCCESS;
3125 }
3126
3127 /* Schedule NAPI for backlog device
3128 * We can use non atomic operation since we own the queue lock
3129 */
3130 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3131 if (!rps_ipi_queued(sd))
3132 ____napi_schedule(sd, &sd->backlog);
3133 }
3134 goto enqueue;
3135 }
3136
3137 sd->dropped++;
3138 rps_unlock(sd);
3139
3140 local_irq_restore(flags);
3141
3142 atomic_long_inc(&skb->dev->rx_dropped);
3143 kfree_skb(skb);
3144 return NET_RX_DROP;
3145 }
3146
3147 /**
3148 * netif_rx - post buffer to the network code
3149 * @skb: buffer to post
3150 *
3151 * This function receives a packet from a device driver and queues it for
3152 * the upper (protocol) levels to process. It always succeeds. The buffer
3153 * may be dropped during processing for congestion control or by the
3154 * protocol layers.
3155 *
3156 * return values:
3157 * NET_RX_SUCCESS (no congestion)
3158 * NET_RX_DROP (packet was dropped)
3159 *
3160 */
3161
netif_rx(struct sk_buff * skb)3162 int netif_rx(struct sk_buff *skb)
3163 {
3164 int ret;
3165
3166 /* if netpoll wants it, pretend we never saw it */
3167 if (netpoll_rx(skb))
3168 return NET_RX_DROP;
3169
3170 net_timestamp_check(netdev_tstamp_prequeue, skb);
3171
3172 trace_netif_rx(skb);
3173 #ifdef CONFIG_RPS
3174 if (static_key_false(&rps_needed)) {
3175 struct rps_dev_flow voidflow, *rflow = &voidflow;
3176 int cpu;
3177
3178 preempt_disable();
3179 rcu_read_lock();
3180
3181 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3182 if (cpu < 0)
3183 cpu = smp_processor_id();
3184
3185 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3186
3187 rcu_read_unlock();
3188 preempt_enable();
3189 } else
3190 #endif
3191 {
3192 unsigned int qtail;
3193 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3194 put_cpu();
3195 }
3196 return ret;
3197 }
3198 EXPORT_SYMBOL(netif_rx);
3199
netif_rx_ni(struct sk_buff * skb)3200 int netif_rx_ni(struct sk_buff *skb)
3201 {
3202 int err;
3203
3204 preempt_disable();
3205 err = netif_rx(skb);
3206 if (local_softirq_pending())
3207 do_softirq();
3208 preempt_enable();
3209
3210 return err;
3211 }
3212 EXPORT_SYMBOL(netif_rx_ni);
3213
net_tx_action(struct softirq_action * h)3214 static void net_tx_action(struct softirq_action *h)
3215 {
3216 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3217
3218 if (sd->completion_queue) {
3219 struct sk_buff *clist;
3220
3221 local_irq_disable();
3222 clist = sd->completion_queue;
3223 sd->completion_queue = NULL;
3224 local_irq_enable();
3225
3226 while (clist) {
3227 struct sk_buff *skb = clist;
3228 clist = clist->next;
3229
3230 WARN_ON(atomic_read(&skb->users));
3231 trace_kfree_skb(skb, net_tx_action);
3232 __kfree_skb(skb);
3233 }
3234 }
3235
3236 if (sd->output_queue) {
3237 struct Qdisc *head;
3238
3239 local_irq_disable();
3240 head = sd->output_queue;
3241 sd->output_queue = NULL;
3242 sd->output_queue_tailp = &sd->output_queue;
3243 local_irq_enable();
3244
3245 while (head) {
3246 struct Qdisc *q = head;
3247 spinlock_t *root_lock;
3248
3249 head = head->next_sched;
3250
3251 root_lock = qdisc_lock(q);
3252 if (spin_trylock(root_lock)) {
3253 smp_mb__before_clear_bit();
3254 clear_bit(__QDISC_STATE_SCHED,
3255 &q->state);
3256 qdisc_run(q);
3257 spin_unlock(root_lock);
3258 } else {
3259 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3260 &q->state)) {
3261 __netif_reschedule(q);
3262 } else {
3263 smp_mb__before_clear_bit();
3264 clear_bit(__QDISC_STATE_SCHED,
3265 &q->state);
3266 }
3267 }
3268 }
3269 }
3270 }
3271
3272 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3273 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3274 /* This hook is defined here for ATM LANE */
3275 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3276 unsigned char *addr) __read_mostly;
3277 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3278 #endif
3279
3280 #ifdef CONFIG_NET_CLS_ACT
3281 /* TODO: Maybe we should just force sch_ingress to be compiled in
3282 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3283 * a compare and 2 stores extra right now if we dont have it on
3284 * but have CONFIG_NET_CLS_ACT
3285 * NOTE: This doesn't stop any functionality; if you dont have
3286 * the ingress scheduler, you just can't add policies on ingress.
3287 *
3288 */
ing_filter(struct sk_buff * skb,struct netdev_queue * rxq)3289 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
3290 {
3291 struct net_device *dev = skb->dev;
3292 u32 ttl = G_TC_RTTL(skb->tc_verd);
3293 int result = TC_ACT_OK;
3294 struct Qdisc *q;
3295
3296 if (unlikely(MAX_RED_LOOP < ttl++)) {
3297 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3298 skb->skb_iif, dev->ifindex);
3299 return TC_ACT_SHOT;
3300 }
3301
3302 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3303 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3304
3305 q = rxq->qdisc;
3306 if (q != &noop_qdisc) {
3307 spin_lock(qdisc_lock(q));
3308 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3309 result = qdisc_enqueue_root(skb, q);
3310 spin_unlock(qdisc_lock(q));
3311 }
3312
3313 return result;
3314 }
3315
handle_ing(struct sk_buff * skb,struct packet_type ** pt_prev,int * ret,struct net_device * orig_dev)3316 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3317 struct packet_type **pt_prev,
3318 int *ret, struct net_device *orig_dev)
3319 {
3320 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3321
3322 if (!rxq || rxq->qdisc == &noop_qdisc)
3323 goto out;
3324
3325 if (*pt_prev) {
3326 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3327 *pt_prev = NULL;
3328 }
3329
3330 switch (ing_filter(skb, rxq)) {
3331 case TC_ACT_SHOT:
3332 case TC_ACT_STOLEN:
3333 kfree_skb(skb);
3334 return NULL;
3335 }
3336
3337 out:
3338 skb->tc_verd = 0;
3339 return skb;
3340 }
3341 #endif
3342
3343 /**
3344 * netdev_rx_handler_register - register receive handler
3345 * @dev: device to register a handler for
3346 * @rx_handler: receive handler to register
3347 * @rx_handler_data: data pointer that is used by rx handler
3348 *
3349 * Register a receive hander for a device. This handler will then be
3350 * called from __netif_receive_skb. A negative errno code is returned
3351 * on a failure.
3352 *
3353 * The caller must hold the rtnl_mutex.
3354 *
3355 * For a general description of rx_handler, see enum rx_handler_result.
3356 */
netdev_rx_handler_register(struct net_device * dev,rx_handler_func_t * rx_handler,void * rx_handler_data)3357 int netdev_rx_handler_register(struct net_device *dev,
3358 rx_handler_func_t *rx_handler,
3359 void *rx_handler_data)
3360 {
3361 ASSERT_RTNL();
3362
3363 if (dev->rx_handler)
3364 return -EBUSY;
3365
3366 /* Note: rx_handler_data must be set before rx_handler */
3367 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3368 rcu_assign_pointer(dev->rx_handler, rx_handler);
3369
3370 return 0;
3371 }
3372 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3373
3374 /**
3375 * netdev_rx_handler_unregister - unregister receive handler
3376 * @dev: device to unregister a handler from
3377 *
3378 * Unregister a receive handler from a device.
3379 *
3380 * The caller must hold the rtnl_mutex.
3381 */
netdev_rx_handler_unregister(struct net_device * dev)3382 void netdev_rx_handler_unregister(struct net_device *dev)
3383 {
3384
3385 ASSERT_RTNL();
3386 RCU_INIT_POINTER(dev->rx_handler, NULL);
3387 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3388 * section has a guarantee to see a non NULL rx_handler_data
3389 * as well.
3390 */
3391 synchronize_net();
3392 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3393 }
3394 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3395
3396 /*
3397 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3398 * the special handling of PFMEMALLOC skbs.
3399 */
skb_pfmemalloc_protocol(struct sk_buff * skb)3400 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3401 {
3402 switch (skb->protocol) {
3403 case __constant_htons(ETH_P_ARP):
3404 case __constant_htons(ETH_P_IP):
3405 case __constant_htons(ETH_P_IPV6):
3406 case __constant_htons(ETH_P_8021Q):
3407 case __constant_htons(ETH_P_8021AD):
3408 return true;
3409 default:
3410 return false;
3411 }
3412 }
3413
__netif_receive_skb_core(struct sk_buff * skb,bool pfmemalloc)3414 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
3415 {
3416 struct packet_type *ptype, *pt_prev;
3417 rx_handler_func_t *rx_handler;
3418 struct net_device *orig_dev;
3419 struct net_device *null_or_dev;
3420 bool deliver_exact = false;
3421 int ret = NET_RX_DROP;
3422 __be16 type;
3423
3424 net_timestamp_check(!netdev_tstamp_prequeue, skb);
3425
3426 trace_netif_receive_skb(skb);
3427
3428 /* if we've gotten here through NAPI, check netpoll */
3429 if (netpoll_receive_skb(skb))
3430 goto out;
3431
3432 orig_dev = skb->dev;
3433
3434 skb_reset_network_header(skb);
3435 if (!skb_transport_header_was_set(skb))
3436 skb_reset_transport_header(skb);
3437 skb_reset_mac_len(skb);
3438
3439 pt_prev = NULL;
3440
3441 rcu_read_lock();
3442
3443 another_round:
3444 skb->skb_iif = skb->dev->ifindex;
3445
3446 __this_cpu_inc(softnet_data.processed);
3447
3448 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3449 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
3450 skb = vlan_untag(skb);
3451 if (unlikely(!skb))
3452 goto unlock;
3453 }
3454
3455 #ifdef CONFIG_NET_CLS_ACT
3456 if (skb->tc_verd & TC_NCLS) {
3457 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3458 goto ncls;
3459 }
3460 #endif
3461
3462 if (pfmemalloc)
3463 goto skip_taps;
3464
3465 list_for_each_entry_rcu(ptype, &ptype_all, list) {
3466 if (!ptype->dev || ptype->dev == skb->dev) {
3467 if (pt_prev)
3468 ret = deliver_skb(skb, pt_prev, orig_dev);
3469 pt_prev = ptype;
3470 }
3471 }
3472
3473 skip_taps:
3474 #ifdef CONFIG_NET_CLS_ACT
3475 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3476 if (!skb)
3477 goto unlock;
3478 ncls:
3479 #endif
3480
3481 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
3482 goto drop;
3483
3484 if (vlan_tx_tag_present(skb)) {
3485 if (pt_prev) {
3486 ret = deliver_skb(skb, pt_prev, orig_dev);
3487 pt_prev = NULL;
3488 }
3489 if (vlan_do_receive(&skb))
3490 goto another_round;
3491 else if (unlikely(!skb))
3492 goto unlock;
3493 }
3494
3495 rx_handler = rcu_dereference(skb->dev->rx_handler);
3496 if (rx_handler) {
3497 if (pt_prev) {
3498 ret = deliver_skb(skb, pt_prev, orig_dev);
3499 pt_prev = NULL;
3500 }
3501 switch (rx_handler(&skb)) {
3502 case RX_HANDLER_CONSUMED:
3503 ret = NET_RX_SUCCESS;
3504 goto unlock;
3505 case RX_HANDLER_ANOTHER:
3506 goto another_round;
3507 case RX_HANDLER_EXACT:
3508 deliver_exact = true;
3509 case RX_HANDLER_PASS:
3510 break;
3511 default:
3512 BUG();
3513 }
3514 }
3515
3516 if (vlan_tx_nonzero_tag_present(skb))
3517 skb->pkt_type = PACKET_OTHERHOST;
3518
3519 /* deliver only exact match when indicated */
3520 null_or_dev = deliver_exact ? skb->dev : NULL;
3521
3522 type = skb->protocol;
3523 list_for_each_entry_rcu(ptype,
3524 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3525 if (ptype->type == type &&
3526 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3527 ptype->dev == orig_dev)) {
3528 if (pt_prev)
3529 ret = deliver_skb(skb, pt_prev, orig_dev);
3530 pt_prev = ptype;
3531 }
3532 }
3533
3534 if (pt_prev) {
3535 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3536 goto drop;
3537 else
3538 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3539 } else {
3540 drop:
3541 atomic_long_inc(&skb->dev->rx_dropped);
3542 kfree_skb(skb);
3543 /* Jamal, now you will not able to escape explaining
3544 * me how you were going to use this. :-)
3545 */
3546 ret = NET_RX_DROP;
3547 }
3548
3549 unlock:
3550 rcu_read_unlock();
3551 out:
3552 return ret;
3553 }
3554
__netif_receive_skb(struct sk_buff * skb)3555 static int __netif_receive_skb(struct sk_buff *skb)
3556 {
3557 int ret;
3558
3559 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3560 unsigned long pflags = current->flags;
3561
3562 /*
3563 * PFMEMALLOC skbs are special, they should
3564 * - be delivered to SOCK_MEMALLOC sockets only
3565 * - stay away from userspace
3566 * - have bounded memory usage
3567 *
3568 * Use PF_MEMALLOC as this saves us from propagating the allocation
3569 * context down to all allocation sites.
3570 */
3571 current->flags |= PF_MEMALLOC;
3572 ret = __netif_receive_skb_core(skb, true);
3573 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3574 } else
3575 ret = __netif_receive_skb_core(skb, false);
3576
3577 return ret;
3578 }
3579
3580 /**
3581 * netif_receive_skb - process receive buffer from network
3582 * @skb: buffer to process
3583 *
3584 * netif_receive_skb() is the main receive data processing function.
3585 * It always succeeds. The buffer may be dropped during processing
3586 * for congestion control or by the protocol layers.
3587 *
3588 * This function may only be called from softirq context and interrupts
3589 * should be enabled.
3590 *
3591 * Return values (usually ignored):
3592 * NET_RX_SUCCESS: no congestion
3593 * NET_RX_DROP: packet was dropped
3594 */
netif_receive_skb(struct sk_buff * skb)3595 int netif_receive_skb(struct sk_buff *skb)
3596 {
3597 net_timestamp_check(netdev_tstamp_prequeue, skb);
3598
3599 if (skb_defer_rx_timestamp(skb))
3600 return NET_RX_SUCCESS;
3601
3602 #ifdef CONFIG_RPS
3603 if (static_key_false(&rps_needed)) {
3604 struct rps_dev_flow voidflow, *rflow = &voidflow;
3605 int cpu, ret;
3606
3607 rcu_read_lock();
3608
3609 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3610
3611 if (cpu >= 0) {
3612 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3613 rcu_read_unlock();
3614 return ret;
3615 }
3616 rcu_read_unlock();
3617 }
3618 #endif
3619 return __netif_receive_skb(skb);
3620 }
3621 EXPORT_SYMBOL(netif_receive_skb);
3622
3623 /* Network device is going away, flush any packets still pending
3624 * Called with irqs disabled.
3625 */
flush_backlog(void * arg)3626 static void flush_backlog(void *arg)
3627 {
3628 struct net_device *dev = arg;
3629 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3630 struct sk_buff *skb, *tmp;
3631
3632 rps_lock(sd);
3633 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3634 if (skb->dev == dev) {
3635 __skb_unlink(skb, &sd->input_pkt_queue);
3636 kfree_skb(skb);
3637 input_queue_head_incr(sd);
3638 }
3639 }
3640 rps_unlock(sd);
3641
3642 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3643 if (skb->dev == dev) {
3644 __skb_unlink(skb, &sd->process_queue);
3645 kfree_skb(skb);
3646 input_queue_head_incr(sd);
3647 }
3648 }
3649 }
3650
napi_gro_complete(struct sk_buff * skb)3651 static int napi_gro_complete(struct sk_buff *skb)
3652 {
3653 struct packet_offload *ptype;
3654 __be16 type = skb->protocol;
3655 struct list_head *head = &offload_base;
3656 int err = -ENOENT;
3657
3658 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
3659
3660 if (NAPI_GRO_CB(skb)->count == 1) {
3661 skb_shinfo(skb)->gso_size = 0;
3662 goto out;
3663 }
3664
3665 rcu_read_lock();
3666 list_for_each_entry_rcu(ptype, head, list) {
3667 if (ptype->type != type || !ptype->callbacks.gro_complete)
3668 continue;
3669
3670 err = ptype->callbacks.gro_complete(skb);
3671 break;
3672 }
3673 rcu_read_unlock();
3674
3675 if (err) {
3676 WARN_ON(&ptype->list == head);
3677 kfree_skb(skb);
3678 return NET_RX_SUCCESS;
3679 }
3680
3681 out:
3682 return netif_receive_skb(skb);
3683 }
3684
3685 /* napi->gro_list contains packets ordered by age.
3686 * youngest packets at the head of it.
3687 * Complete skbs in reverse order to reduce latencies.
3688 */
napi_gro_flush(struct napi_struct * napi,bool flush_old)3689 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
3690 {
3691 struct sk_buff *skb, *prev = NULL;
3692
3693 /* scan list and build reverse chain */
3694 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3695 skb->prev = prev;
3696 prev = skb;
3697 }
3698
3699 for (skb = prev; skb; skb = prev) {
3700 skb->next = NULL;
3701
3702 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3703 return;
3704
3705 prev = skb->prev;
3706 napi_gro_complete(skb);
3707 napi->gro_count--;
3708 }
3709
3710 napi->gro_list = NULL;
3711 }
3712 EXPORT_SYMBOL(napi_gro_flush);
3713
gro_list_prepare(struct napi_struct * napi,struct sk_buff * skb)3714 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
3715 {
3716 struct sk_buff *p;
3717 unsigned int maclen = skb->dev->hard_header_len;
3718
3719 for (p = napi->gro_list; p; p = p->next) {
3720 unsigned long diffs;
3721
3722 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3723 diffs |= p->vlan_tci ^ skb->vlan_tci;
3724 if (maclen == ETH_HLEN)
3725 diffs |= compare_ether_header(skb_mac_header(p),
3726 skb_gro_mac_header(skb));
3727 else if (!diffs)
3728 diffs = memcmp(skb_mac_header(p),
3729 skb_gro_mac_header(skb),
3730 maclen);
3731 NAPI_GRO_CB(p)->same_flow = !diffs;
3732 NAPI_GRO_CB(p)->flush = 0;
3733 }
3734 }
3735
dev_gro_receive(struct napi_struct * napi,struct sk_buff * skb)3736 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3737 {
3738 struct sk_buff **pp = NULL;
3739 struct packet_offload *ptype;
3740 __be16 type = skb->protocol;
3741 struct list_head *head = &offload_base;
3742 int same_flow;
3743 enum gro_result ret;
3744
3745 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3746 goto normal;
3747
3748 if (skb_is_gso(skb) || skb_has_frag_list(skb))
3749 goto normal;
3750
3751 gro_list_prepare(napi, skb);
3752
3753 rcu_read_lock();
3754 list_for_each_entry_rcu(ptype, head, list) {
3755 if (ptype->type != type || !ptype->callbacks.gro_receive)
3756 continue;
3757
3758 skb_set_network_header(skb, skb_gro_offset(skb));
3759 skb_reset_mac_len(skb);
3760 NAPI_GRO_CB(skb)->same_flow = 0;
3761 NAPI_GRO_CB(skb)->flush = 0;
3762 NAPI_GRO_CB(skb)->free = 0;
3763
3764 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
3765 break;
3766 }
3767 rcu_read_unlock();
3768
3769 if (&ptype->list == head)
3770 goto normal;
3771
3772 same_flow = NAPI_GRO_CB(skb)->same_flow;
3773 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3774
3775 if (pp) {
3776 struct sk_buff *nskb = *pp;
3777
3778 *pp = nskb->next;
3779 nskb->next = NULL;
3780 napi_gro_complete(nskb);
3781 napi->gro_count--;
3782 }
3783
3784 if (same_flow)
3785 goto ok;
3786
3787 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3788 goto normal;
3789
3790 napi->gro_count++;
3791 NAPI_GRO_CB(skb)->count = 1;
3792 NAPI_GRO_CB(skb)->age = jiffies;
3793 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3794 skb->next = napi->gro_list;
3795 napi->gro_list = skb;
3796 ret = GRO_HELD;
3797
3798 pull:
3799 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3800 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3801
3802 BUG_ON(skb->end - skb->tail < grow);
3803
3804 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3805
3806 skb->tail += grow;
3807 skb->data_len -= grow;
3808
3809 skb_shinfo(skb)->frags[0].page_offset += grow;
3810 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
3811
3812 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
3813 skb_frag_unref(skb, 0);
3814 memmove(skb_shinfo(skb)->frags,
3815 skb_shinfo(skb)->frags + 1,
3816 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3817 }
3818 }
3819
3820 ok:
3821 return ret;
3822
3823 normal:
3824 ret = GRO_NORMAL;
3825 goto pull;
3826 }
3827
3828
napi_skb_finish(gro_result_t ret,struct sk_buff * skb)3829 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3830 {
3831 switch (ret) {
3832 case GRO_NORMAL:
3833 if (netif_receive_skb(skb))
3834 ret = GRO_DROP;
3835 break;
3836
3837 case GRO_DROP:
3838 kfree_skb(skb);
3839 break;
3840
3841 case GRO_MERGED_FREE:
3842 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3843 kmem_cache_free(skbuff_head_cache, skb);
3844 else
3845 __kfree_skb(skb);
3846 break;
3847
3848 case GRO_HELD:
3849 case GRO_MERGED:
3850 break;
3851 }
3852
3853 return ret;
3854 }
3855
skb_gro_reset_offset(struct sk_buff * skb)3856 static void skb_gro_reset_offset(struct sk_buff *skb)
3857 {
3858 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3859 const skb_frag_t *frag0 = &pinfo->frags[0];
3860
3861 NAPI_GRO_CB(skb)->data_offset = 0;
3862 NAPI_GRO_CB(skb)->frag0 = NULL;
3863 NAPI_GRO_CB(skb)->frag0_len = 0;
3864
3865 if (skb->mac_header == skb->tail &&
3866 pinfo->nr_frags &&
3867 !PageHighMem(skb_frag_page(frag0))) {
3868 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3869 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
3870 }
3871 }
3872
napi_gro_receive(struct napi_struct * napi,struct sk_buff * skb)3873 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3874 {
3875 skb_gro_reset_offset(skb);
3876
3877 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
3878 }
3879 EXPORT_SYMBOL(napi_gro_receive);
3880
napi_reuse_skb(struct napi_struct * napi,struct sk_buff * skb)3881 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3882 {
3883 __skb_pull(skb, skb_headlen(skb));
3884 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3885 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
3886 skb->vlan_tci = 0;
3887 skb->dev = napi->dev;
3888 skb->skb_iif = 0;
3889
3890 napi->skb = skb;
3891 }
3892
napi_get_frags(struct napi_struct * napi)3893 struct sk_buff *napi_get_frags(struct napi_struct *napi)
3894 {
3895 struct sk_buff *skb = napi->skb;
3896
3897 if (!skb) {
3898 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3899 if (skb)
3900 napi->skb = skb;
3901 }
3902 return skb;
3903 }
3904 EXPORT_SYMBOL(napi_get_frags);
3905
napi_frags_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)3906 static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3907 gro_result_t ret)
3908 {
3909 switch (ret) {
3910 case GRO_NORMAL:
3911 case GRO_HELD:
3912 skb->protocol = eth_type_trans(skb, skb->dev);
3913
3914 if (ret == GRO_HELD)
3915 skb_gro_pull(skb, -ETH_HLEN);
3916 else if (netif_receive_skb(skb))
3917 ret = GRO_DROP;
3918 break;
3919
3920 case GRO_DROP:
3921 case GRO_MERGED_FREE:
3922 napi_reuse_skb(napi, skb);
3923 break;
3924
3925 case GRO_MERGED:
3926 break;
3927 }
3928
3929 return ret;
3930 }
3931
napi_frags_skb(struct napi_struct * napi)3932 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3933 {
3934 struct sk_buff *skb = napi->skb;
3935 struct ethhdr *eth;
3936 unsigned int hlen;
3937 unsigned int off;
3938
3939 napi->skb = NULL;
3940
3941 skb_reset_mac_header(skb);
3942 skb_gro_reset_offset(skb);
3943
3944 off = skb_gro_offset(skb);
3945 hlen = off + sizeof(*eth);
3946 eth = skb_gro_header_fast(skb, off);
3947 if (skb_gro_header_hard(skb, hlen)) {
3948 eth = skb_gro_header_slow(skb, hlen, off);
3949 if (unlikely(!eth)) {
3950 napi_reuse_skb(napi, skb);
3951 skb = NULL;
3952 goto out;
3953 }
3954 }
3955
3956 skb_gro_pull(skb, sizeof(*eth));
3957
3958 /*
3959 * This works because the only protocols we care about don't require
3960 * special handling. We'll fix it up properly at the end.
3961 */
3962 skb->protocol = eth->h_proto;
3963
3964 out:
3965 return skb;
3966 }
3967
napi_gro_frags(struct napi_struct * napi)3968 gro_result_t napi_gro_frags(struct napi_struct *napi)
3969 {
3970 struct sk_buff *skb = napi_frags_skb(napi);
3971
3972 if (!skb)
3973 return GRO_DROP;
3974
3975 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
3976 }
3977 EXPORT_SYMBOL(napi_gro_frags);
3978
3979 /*
3980 * net_rps_action sends any pending IPI's for rps.
3981 * Note: called with local irq disabled, but exits with local irq enabled.
3982 */
net_rps_action_and_irq_enable(struct softnet_data * sd)3983 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3984 {
3985 #ifdef CONFIG_RPS
3986 struct softnet_data *remsd = sd->rps_ipi_list;
3987
3988 if (remsd) {
3989 sd->rps_ipi_list = NULL;
3990
3991 local_irq_enable();
3992
3993 /* Send pending IPI's to kick RPS processing on remote cpus. */
3994 while (remsd) {
3995 struct softnet_data *next = remsd->rps_ipi_next;
3996
3997 if (cpu_online(remsd->cpu))
3998 __smp_call_function_single(remsd->cpu,
3999 &remsd->csd, 0);
4000 remsd = next;
4001 }
4002 } else
4003 #endif
4004 local_irq_enable();
4005 }
4006
process_backlog(struct napi_struct * napi,int quota)4007 static int process_backlog(struct napi_struct *napi, int quota)
4008 {
4009 int work = 0;
4010 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4011
4012 #ifdef CONFIG_RPS
4013 /* Check if we have pending ipi, its better to send them now,
4014 * not waiting net_rx_action() end.
4015 */
4016 if (sd->rps_ipi_list) {
4017 local_irq_disable();
4018 net_rps_action_and_irq_enable(sd);
4019 }
4020 #endif
4021 napi->weight = weight_p;
4022 local_irq_disable();
4023 while (work < quota) {
4024 struct sk_buff *skb;
4025 unsigned int qlen;
4026
4027 while ((skb = __skb_dequeue(&sd->process_queue))) {
4028 local_irq_enable();
4029 __netif_receive_skb(skb);
4030 local_irq_disable();
4031 input_queue_head_incr(sd);
4032 if (++work >= quota) {
4033 local_irq_enable();
4034 return work;
4035 }
4036 }
4037
4038 rps_lock(sd);
4039 qlen = skb_queue_len(&sd->input_pkt_queue);
4040 if (qlen)
4041 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4042 &sd->process_queue);
4043
4044 if (qlen < quota - work) {
4045 /*
4046 * Inline a custom version of __napi_complete().
4047 * only current cpu owns and manipulates this napi,
4048 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
4049 * we can use a plain write instead of clear_bit(),
4050 * and we dont need an smp_mb() memory barrier.
4051 */
4052 list_del(&napi->poll_list);
4053 napi->state = 0;
4054
4055 quota = work + qlen;
4056 }
4057 rps_unlock(sd);
4058 }
4059 local_irq_enable();
4060
4061 return work;
4062 }
4063
4064 /**
4065 * __napi_schedule - schedule for receive
4066 * @n: entry to schedule
4067 *
4068 * The entry's receive function will be scheduled to run
4069 */
__napi_schedule(struct napi_struct * n)4070 void __napi_schedule(struct napi_struct *n)
4071 {
4072 unsigned long flags;
4073
4074 local_irq_save(flags);
4075 ____napi_schedule(&__get_cpu_var(softnet_data), n);
4076 local_irq_restore(flags);
4077 }
4078 EXPORT_SYMBOL(__napi_schedule);
4079
__napi_complete(struct napi_struct * n)4080 void __napi_complete(struct napi_struct *n)
4081 {
4082 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4083 BUG_ON(n->gro_list);
4084
4085 list_del(&n->poll_list);
4086 smp_mb__before_clear_bit();
4087 clear_bit(NAPI_STATE_SCHED, &n->state);
4088 }
4089 EXPORT_SYMBOL(__napi_complete);
4090
napi_complete(struct napi_struct * n)4091 void napi_complete(struct napi_struct *n)
4092 {
4093 unsigned long flags;
4094
4095 /*
4096 * don't let napi dequeue from the cpu poll list
4097 * just in case its running on a different cpu
4098 */
4099 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4100 return;
4101
4102 napi_gro_flush(n, false);
4103 local_irq_save(flags);
4104 __napi_complete(n);
4105 local_irq_restore(flags);
4106 }
4107 EXPORT_SYMBOL(napi_complete);
4108
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)4109 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4110 int (*poll)(struct napi_struct *, int), int weight)
4111 {
4112 INIT_LIST_HEAD(&napi->poll_list);
4113 napi->gro_count = 0;
4114 napi->gro_list = NULL;
4115 napi->skb = NULL;
4116 napi->poll = poll;
4117 if (weight > NAPI_POLL_WEIGHT)
4118 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4119 weight, dev->name);
4120 napi->weight = weight;
4121 list_add(&napi->dev_list, &dev->napi_list);
4122 napi->dev = dev;
4123 #ifdef CONFIG_NETPOLL
4124 spin_lock_init(&napi->poll_lock);
4125 napi->poll_owner = -1;
4126 #endif
4127 set_bit(NAPI_STATE_SCHED, &napi->state);
4128 }
4129 EXPORT_SYMBOL(netif_napi_add);
4130
netif_napi_del(struct napi_struct * napi)4131 void netif_napi_del(struct napi_struct *napi)
4132 {
4133 struct sk_buff *skb, *next;
4134
4135 list_del_init(&napi->dev_list);
4136 napi_free_frags(napi);
4137
4138 for (skb = napi->gro_list; skb; skb = next) {
4139 next = skb->next;
4140 skb->next = NULL;
4141 kfree_skb(skb);
4142 }
4143
4144 napi->gro_list = NULL;
4145 napi->gro_count = 0;
4146 }
4147 EXPORT_SYMBOL(netif_napi_del);
4148
net_rx_action(struct softirq_action * h)4149 static void net_rx_action(struct softirq_action *h)
4150 {
4151 struct softnet_data *sd = &__get_cpu_var(softnet_data);
4152 unsigned long time_limit = jiffies + 2;
4153 int budget = netdev_budget;
4154 void *have;
4155
4156 local_irq_disable();
4157
4158 while (!list_empty(&sd->poll_list)) {
4159 struct napi_struct *n;
4160 int work, weight;
4161
4162 /* If softirq window is exhuasted then punt.
4163 * Allow this to run for 2 jiffies since which will allow
4164 * an average latency of 1.5/HZ.
4165 */
4166 if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit)))
4167 goto softnet_break;
4168
4169 local_irq_enable();
4170
4171 /* Even though interrupts have been re-enabled, this
4172 * access is safe because interrupts can only add new
4173 * entries to the tail of this list, and only ->poll()
4174 * calls can remove this head entry from the list.
4175 */
4176 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
4177
4178 have = netpoll_poll_lock(n);
4179
4180 weight = n->weight;
4181
4182 /* This NAPI_STATE_SCHED test is for avoiding a race
4183 * with netpoll's poll_napi(). Only the entity which
4184 * obtains the lock and sees NAPI_STATE_SCHED set will
4185 * actually make the ->poll() call. Therefore we avoid
4186 * accidentally calling ->poll() when NAPI is not scheduled.
4187 */
4188 work = 0;
4189 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4190 work = n->poll(n, weight);
4191 trace_napi_poll(n);
4192 }
4193
4194 WARN_ON_ONCE(work > weight);
4195
4196 budget -= work;
4197
4198 local_irq_disable();
4199
4200 /* Drivers must not modify the NAPI state if they
4201 * consume the entire weight. In such cases this code
4202 * still "owns" the NAPI instance and therefore can
4203 * move the instance around on the list at-will.
4204 */
4205 if (unlikely(work == weight)) {
4206 if (unlikely(napi_disable_pending(n))) {
4207 local_irq_enable();
4208 napi_complete(n);
4209 local_irq_disable();
4210 } else {
4211 if (n->gro_list) {
4212 /* flush too old packets
4213 * If HZ < 1000, flush all packets.
4214 */
4215 local_irq_enable();
4216 napi_gro_flush(n, HZ >= 1000);
4217 local_irq_disable();
4218 }
4219 list_move_tail(&n->poll_list, &sd->poll_list);
4220 }
4221 }
4222
4223 netpoll_poll_unlock(have);
4224 }
4225 out:
4226 net_rps_action_and_irq_enable(sd);
4227
4228 #ifdef CONFIG_NET_DMA
4229 /*
4230 * There may not be any more sk_buffs coming right now, so push
4231 * any pending DMA copies to hardware
4232 */
4233 dma_issue_pending_all();
4234 #endif
4235
4236 return;
4237
4238 softnet_break:
4239 sd->time_squeeze++;
4240 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4241 goto out;
4242 }
4243
4244 struct netdev_upper {
4245 struct net_device *dev;
4246 bool master;
4247 struct list_head list;
4248 struct rcu_head rcu;
4249 struct list_head search_list;
4250 };
4251
__append_search_uppers(struct list_head * search_list,struct net_device * dev)4252 static void __append_search_uppers(struct list_head *search_list,
4253 struct net_device *dev)
4254 {
4255 struct netdev_upper *upper;
4256
4257 list_for_each_entry(upper, &dev->upper_dev_list, list) {
4258 /* check if this upper is not already in search list */
4259 if (list_empty(&upper->search_list))
4260 list_add_tail(&upper->search_list, search_list);
4261 }
4262 }
4263
__netdev_search_upper_dev(struct net_device * dev,struct net_device * upper_dev)4264 static bool __netdev_search_upper_dev(struct net_device *dev,
4265 struct net_device *upper_dev)
4266 {
4267 LIST_HEAD(search_list);
4268 struct netdev_upper *upper;
4269 struct netdev_upper *tmp;
4270 bool ret = false;
4271
4272 __append_search_uppers(&search_list, dev);
4273 list_for_each_entry(upper, &search_list, search_list) {
4274 if (upper->dev == upper_dev) {
4275 ret = true;
4276 break;
4277 }
4278 __append_search_uppers(&search_list, upper->dev);
4279 }
4280 list_for_each_entry_safe(upper, tmp, &search_list, search_list)
4281 INIT_LIST_HEAD(&upper->search_list);
4282 return ret;
4283 }
4284
__netdev_find_upper(struct net_device * dev,struct net_device * upper_dev)4285 static struct netdev_upper *__netdev_find_upper(struct net_device *dev,
4286 struct net_device *upper_dev)
4287 {
4288 struct netdev_upper *upper;
4289
4290 list_for_each_entry(upper, &dev->upper_dev_list, list) {
4291 if (upper->dev == upper_dev)
4292 return upper;
4293 }
4294 return NULL;
4295 }
4296
4297 /**
4298 * netdev_has_upper_dev - Check if device is linked to an upper device
4299 * @dev: device
4300 * @upper_dev: upper device to check
4301 *
4302 * Find out if a device is linked to specified upper device and return true
4303 * in case it is. Note that this checks only immediate upper device,
4304 * not through a complete stack of devices. The caller must hold the RTNL lock.
4305 */
netdev_has_upper_dev(struct net_device * dev,struct net_device * upper_dev)4306 bool netdev_has_upper_dev(struct net_device *dev,
4307 struct net_device *upper_dev)
4308 {
4309 ASSERT_RTNL();
4310
4311 return __netdev_find_upper(dev, upper_dev);
4312 }
4313 EXPORT_SYMBOL(netdev_has_upper_dev);
4314
4315 /**
4316 * netdev_has_any_upper_dev - Check if device is linked to some device
4317 * @dev: device
4318 *
4319 * Find out if a device is linked to an upper device and return true in case
4320 * it is. The caller must hold the RTNL lock.
4321 */
netdev_has_any_upper_dev(struct net_device * dev)4322 bool netdev_has_any_upper_dev(struct net_device *dev)
4323 {
4324 ASSERT_RTNL();
4325
4326 return !list_empty(&dev->upper_dev_list);
4327 }
4328 EXPORT_SYMBOL(netdev_has_any_upper_dev);
4329
4330 /**
4331 * netdev_master_upper_dev_get - Get master upper device
4332 * @dev: device
4333 *
4334 * Find a master upper device and return pointer to it or NULL in case
4335 * it's not there. The caller must hold the RTNL lock.
4336 */
netdev_master_upper_dev_get(struct net_device * dev)4337 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4338 {
4339 struct netdev_upper *upper;
4340
4341 ASSERT_RTNL();
4342
4343 if (list_empty(&dev->upper_dev_list))
4344 return NULL;
4345
4346 upper = list_first_entry(&dev->upper_dev_list,
4347 struct netdev_upper, list);
4348 if (likely(upper->master))
4349 return upper->dev;
4350 return NULL;
4351 }
4352 EXPORT_SYMBOL(netdev_master_upper_dev_get);
4353
4354 /**
4355 * netdev_master_upper_dev_get_rcu - Get master upper device
4356 * @dev: device
4357 *
4358 * Find a master upper device and return pointer to it or NULL in case
4359 * it's not there. The caller must hold the RCU read lock.
4360 */
netdev_master_upper_dev_get_rcu(struct net_device * dev)4361 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
4362 {
4363 struct netdev_upper *upper;
4364
4365 upper = list_first_or_null_rcu(&dev->upper_dev_list,
4366 struct netdev_upper, list);
4367 if (upper && likely(upper->master))
4368 return upper->dev;
4369 return NULL;
4370 }
4371 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
4372
__netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev,bool master)4373 static int __netdev_upper_dev_link(struct net_device *dev,
4374 struct net_device *upper_dev, bool master)
4375 {
4376 struct netdev_upper *upper;
4377
4378 ASSERT_RTNL();
4379
4380 if (dev == upper_dev)
4381 return -EBUSY;
4382
4383 /* To prevent loops, check if dev is not upper device to upper_dev. */
4384 if (__netdev_search_upper_dev(upper_dev, dev))
4385 return -EBUSY;
4386
4387 if (__netdev_find_upper(dev, upper_dev))
4388 return -EEXIST;
4389
4390 if (master && netdev_master_upper_dev_get(dev))
4391 return -EBUSY;
4392
4393 upper = kmalloc(sizeof(*upper), GFP_KERNEL);
4394 if (!upper)
4395 return -ENOMEM;
4396
4397 upper->dev = upper_dev;
4398 upper->master = master;
4399 INIT_LIST_HEAD(&upper->search_list);
4400
4401 /* Ensure that master upper link is always the first item in list. */
4402 if (master)
4403 list_add_rcu(&upper->list, &dev->upper_dev_list);
4404 else
4405 list_add_tail_rcu(&upper->list, &dev->upper_dev_list);
4406 dev_hold(upper_dev);
4407
4408 return 0;
4409 }
4410
4411 /**
4412 * netdev_upper_dev_link - Add a link to the upper device
4413 * @dev: device
4414 * @upper_dev: new upper device
4415 *
4416 * Adds a link to device which is upper to this one. The caller must hold
4417 * the RTNL lock. On a failure a negative errno code is returned.
4418 * On success the reference counts are adjusted and the function
4419 * returns zero.
4420 */
netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev)4421 int netdev_upper_dev_link(struct net_device *dev,
4422 struct net_device *upper_dev)
4423 {
4424 return __netdev_upper_dev_link(dev, upper_dev, false);
4425 }
4426 EXPORT_SYMBOL(netdev_upper_dev_link);
4427
4428 /**
4429 * netdev_master_upper_dev_link - Add a master link to the upper device
4430 * @dev: device
4431 * @upper_dev: new upper device
4432 *
4433 * Adds a link to device which is upper to this one. In this case, only
4434 * one master upper device can be linked, although other non-master devices
4435 * might be linked as well. The caller must hold the RTNL lock.
4436 * On a failure a negative errno code is returned. On success the reference
4437 * counts are adjusted and the function returns zero.
4438 */
netdev_master_upper_dev_link(struct net_device * dev,struct net_device * upper_dev)4439 int netdev_master_upper_dev_link(struct net_device *dev,
4440 struct net_device *upper_dev)
4441 {
4442 return __netdev_upper_dev_link(dev, upper_dev, true);
4443 }
4444 EXPORT_SYMBOL(netdev_master_upper_dev_link);
4445
4446 /**
4447 * netdev_upper_dev_unlink - Removes a link to upper device
4448 * @dev: device
4449 * @upper_dev: new upper device
4450 *
4451 * Removes a link to device which is upper to this one. The caller must hold
4452 * the RTNL lock.
4453 */
netdev_upper_dev_unlink(struct net_device * dev,struct net_device * upper_dev)4454 void netdev_upper_dev_unlink(struct net_device *dev,
4455 struct net_device *upper_dev)
4456 {
4457 struct netdev_upper *upper;
4458
4459 ASSERT_RTNL();
4460
4461 upper = __netdev_find_upper(dev, upper_dev);
4462 if (!upper)
4463 return;
4464 list_del_rcu(&upper->list);
4465 dev_put(upper_dev);
4466 kfree_rcu(upper, rcu);
4467 }
4468 EXPORT_SYMBOL(netdev_upper_dev_unlink);
4469
dev_change_rx_flags(struct net_device * dev,int flags)4470 static void dev_change_rx_flags(struct net_device *dev, int flags)
4471 {
4472 const struct net_device_ops *ops = dev->netdev_ops;
4473
4474 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4475 ops->ndo_change_rx_flags(dev, flags);
4476 }
4477
__dev_set_promiscuity(struct net_device * dev,int inc)4478 static int __dev_set_promiscuity(struct net_device *dev, int inc)
4479 {
4480 unsigned int old_flags = dev->flags;
4481 kuid_t uid;
4482 kgid_t gid;
4483
4484 ASSERT_RTNL();
4485
4486 dev->flags |= IFF_PROMISC;
4487 dev->promiscuity += inc;
4488 if (dev->promiscuity == 0) {
4489 /*
4490 * Avoid overflow.
4491 * If inc causes overflow, untouch promisc and return error.
4492 */
4493 if (inc < 0)
4494 dev->flags &= ~IFF_PROMISC;
4495 else {
4496 dev->promiscuity -= inc;
4497 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4498 dev->name);
4499 return -EOVERFLOW;
4500 }
4501 }
4502 if (dev->flags != old_flags) {
4503 pr_info("device %s %s promiscuous mode\n",
4504 dev->name,
4505 dev->flags & IFF_PROMISC ? "entered" : "left");
4506 if (audit_enabled) {
4507 current_uid_gid(&uid, &gid);
4508 audit_log(current->audit_context, GFP_ATOMIC,
4509 AUDIT_ANOM_PROMISCUOUS,
4510 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4511 dev->name, (dev->flags & IFF_PROMISC),
4512 (old_flags & IFF_PROMISC),
4513 from_kuid(&init_user_ns, audit_get_loginuid(current)),
4514 from_kuid(&init_user_ns, uid),
4515 from_kgid(&init_user_ns, gid),
4516 audit_get_sessionid(current));
4517 }
4518
4519 dev_change_rx_flags(dev, IFF_PROMISC);
4520 }
4521 return 0;
4522 }
4523
4524 /**
4525 * dev_set_promiscuity - update promiscuity count on a device
4526 * @dev: device
4527 * @inc: modifier
4528 *
4529 * Add or remove promiscuity from a device. While the count in the device
4530 * remains above zero the interface remains promiscuous. Once it hits zero
4531 * the device reverts back to normal filtering operation. A negative inc
4532 * value is used to drop promiscuity on the device.
4533 * Return 0 if successful or a negative errno code on error.
4534 */
dev_set_promiscuity(struct net_device * dev,int inc)4535 int dev_set_promiscuity(struct net_device *dev, int inc)
4536 {
4537 unsigned int old_flags = dev->flags;
4538 int err;
4539
4540 err = __dev_set_promiscuity(dev, inc);
4541 if (err < 0)
4542 return err;
4543 if (dev->flags != old_flags)
4544 dev_set_rx_mode(dev);
4545 return err;
4546 }
4547 EXPORT_SYMBOL(dev_set_promiscuity);
4548
4549 /**
4550 * dev_set_allmulti - update allmulti count on a device
4551 * @dev: device
4552 * @inc: modifier
4553 *
4554 * Add or remove reception of all multicast frames to a device. While the
4555 * count in the device remains above zero the interface remains listening
4556 * to all interfaces. Once it hits zero the device reverts back to normal
4557 * filtering operation. A negative @inc value is used to drop the counter
4558 * when releasing a resource needing all multicasts.
4559 * Return 0 if successful or a negative errno code on error.
4560 */
4561
dev_set_allmulti(struct net_device * dev,int inc)4562 int dev_set_allmulti(struct net_device *dev, int inc)
4563 {
4564 unsigned int old_flags = dev->flags;
4565
4566 ASSERT_RTNL();
4567
4568 dev->flags |= IFF_ALLMULTI;
4569 dev->allmulti += inc;
4570 if (dev->allmulti == 0) {
4571 /*
4572 * Avoid overflow.
4573 * If inc causes overflow, untouch allmulti and return error.
4574 */
4575 if (inc < 0)
4576 dev->flags &= ~IFF_ALLMULTI;
4577 else {
4578 dev->allmulti -= inc;
4579 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4580 dev->name);
4581 return -EOVERFLOW;
4582 }
4583 }
4584 if (dev->flags ^ old_flags) {
4585 dev_change_rx_flags(dev, IFF_ALLMULTI);
4586 dev_set_rx_mode(dev);
4587 }
4588 return 0;
4589 }
4590 EXPORT_SYMBOL(dev_set_allmulti);
4591
4592 /*
4593 * Upload unicast and multicast address lists to device and
4594 * configure RX filtering. When the device doesn't support unicast
4595 * filtering it is put in promiscuous mode while unicast addresses
4596 * are present.
4597 */
__dev_set_rx_mode(struct net_device * dev)4598 void __dev_set_rx_mode(struct net_device *dev)
4599 {
4600 const struct net_device_ops *ops = dev->netdev_ops;
4601
4602 /* dev_open will call this function so the list will stay sane. */
4603 if (!(dev->flags&IFF_UP))
4604 return;
4605
4606 if (!netif_device_present(dev))
4607 return;
4608
4609 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
4610 /* Unicast addresses changes may only happen under the rtnl,
4611 * therefore calling __dev_set_promiscuity here is safe.
4612 */
4613 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4614 __dev_set_promiscuity(dev, 1);
4615 dev->uc_promisc = true;
4616 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4617 __dev_set_promiscuity(dev, -1);
4618 dev->uc_promisc = false;
4619 }
4620 }
4621
4622 if (ops->ndo_set_rx_mode)
4623 ops->ndo_set_rx_mode(dev);
4624 }
4625
dev_set_rx_mode(struct net_device * dev)4626 void dev_set_rx_mode(struct net_device *dev)
4627 {
4628 netif_addr_lock_bh(dev);
4629 __dev_set_rx_mode(dev);
4630 netif_addr_unlock_bh(dev);
4631 }
4632
4633 /**
4634 * dev_get_flags - get flags reported to userspace
4635 * @dev: device
4636 *
4637 * Get the combination of flag bits exported through APIs to userspace.
4638 */
dev_get_flags(const struct net_device * dev)4639 unsigned int dev_get_flags(const struct net_device *dev)
4640 {
4641 unsigned int flags;
4642
4643 flags = (dev->flags & ~(IFF_PROMISC |
4644 IFF_ALLMULTI |
4645 IFF_RUNNING |
4646 IFF_LOWER_UP |
4647 IFF_DORMANT)) |
4648 (dev->gflags & (IFF_PROMISC |
4649 IFF_ALLMULTI));
4650
4651 if (netif_running(dev)) {
4652 if (netif_oper_up(dev))
4653 flags |= IFF_RUNNING;
4654 if (netif_carrier_ok(dev))
4655 flags |= IFF_LOWER_UP;
4656 if (netif_dormant(dev))
4657 flags |= IFF_DORMANT;
4658 }
4659
4660 return flags;
4661 }
4662 EXPORT_SYMBOL(dev_get_flags);
4663
__dev_change_flags(struct net_device * dev,unsigned int flags)4664 int __dev_change_flags(struct net_device *dev, unsigned int flags)
4665 {
4666 unsigned int old_flags = dev->flags;
4667 int ret;
4668
4669 ASSERT_RTNL();
4670
4671 /*
4672 * Set the flags on our device.
4673 */
4674
4675 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4676 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4677 IFF_AUTOMEDIA)) |
4678 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4679 IFF_ALLMULTI));
4680
4681 /*
4682 * Load in the correct multicast list now the flags have changed.
4683 */
4684
4685 if ((old_flags ^ flags) & IFF_MULTICAST)
4686 dev_change_rx_flags(dev, IFF_MULTICAST);
4687
4688 dev_set_rx_mode(dev);
4689
4690 /*
4691 * Have we downed the interface. We handle IFF_UP ourselves
4692 * according to user attempts to set it, rather than blindly
4693 * setting it.
4694 */
4695
4696 ret = 0;
4697 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
4698 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4699
4700 if (!ret)
4701 dev_set_rx_mode(dev);
4702 }
4703
4704 if ((flags ^ dev->gflags) & IFF_PROMISC) {
4705 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4706
4707 dev->gflags ^= IFF_PROMISC;
4708 dev_set_promiscuity(dev, inc);
4709 }
4710
4711 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4712 is important. Some (broken) drivers set IFF_PROMISC, when
4713 IFF_ALLMULTI is requested not asking us and not reporting.
4714 */
4715 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4716 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4717
4718 dev->gflags ^= IFF_ALLMULTI;
4719 dev_set_allmulti(dev, inc);
4720 }
4721
4722 return ret;
4723 }
4724
__dev_notify_flags(struct net_device * dev,unsigned int old_flags)4725 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4726 {
4727 unsigned int changes = dev->flags ^ old_flags;
4728
4729 if (changes & IFF_UP) {
4730 if (dev->flags & IFF_UP)
4731 call_netdevice_notifiers(NETDEV_UP, dev);
4732 else
4733 call_netdevice_notifiers(NETDEV_DOWN, dev);
4734 }
4735
4736 if (dev->flags & IFF_UP &&
4737 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4738 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4739 }
4740
4741 /**
4742 * dev_change_flags - change device settings
4743 * @dev: device
4744 * @flags: device state flags
4745 *
4746 * Change settings on device based state flags. The flags are
4747 * in the userspace exported format.
4748 */
dev_change_flags(struct net_device * dev,unsigned int flags)4749 int dev_change_flags(struct net_device *dev, unsigned int flags)
4750 {
4751 int ret;
4752 unsigned int changes, old_flags = dev->flags;
4753
4754 ret = __dev_change_flags(dev, flags);
4755 if (ret < 0)
4756 return ret;
4757
4758 changes = old_flags ^ dev->flags;
4759 if (changes)
4760 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4761
4762 __dev_notify_flags(dev, old_flags);
4763 return ret;
4764 }
4765 EXPORT_SYMBOL(dev_change_flags);
4766
4767 /**
4768 * dev_set_mtu - Change maximum transfer unit
4769 * @dev: device
4770 * @new_mtu: new transfer unit
4771 *
4772 * Change the maximum transfer size of the network device.
4773 */
dev_set_mtu(struct net_device * dev,int new_mtu)4774 int dev_set_mtu(struct net_device *dev, int new_mtu)
4775 {
4776 const struct net_device_ops *ops = dev->netdev_ops;
4777 int err;
4778
4779 if (new_mtu == dev->mtu)
4780 return 0;
4781
4782 /* MTU must be positive. */
4783 if (new_mtu < 0)
4784 return -EINVAL;
4785
4786 if (!netif_device_present(dev))
4787 return -ENODEV;
4788
4789 err = 0;
4790 if (ops->ndo_change_mtu)
4791 err = ops->ndo_change_mtu(dev, new_mtu);
4792 else
4793 dev->mtu = new_mtu;
4794
4795 if (!err)
4796 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4797 return err;
4798 }
4799 EXPORT_SYMBOL(dev_set_mtu);
4800
4801 /**
4802 * dev_set_group - Change group this device belongs to
4803 * @dev: device
4804 * @new_group: group this device should belong to
4805 */
dev_set_group(struct net_device * dev,int new_group)4806 void dev_set_group(struct net_device *dev, int new_group)
4807 {
4808 dev->group = new_group;
4809 }
4810 EXPORT_SYMBOL(dev_set_group);
4811
4812 /**
4813 * dev_set_mac_address - Change Media Access Control Address
4814 * @dev: device
4815 * @sa: new address
4816 *
4817 * Change the hardware (MAC) address of the device
4818 */
dev_set_mac_address(struct net_device * dev,struct sockaddr * sa)4819 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4820 {
4821 const struct net_device_ops *ops = dev->netdev_ops;
4822 int err;
4823
4824 if (!ops->ndo_set_mac_address)
4825 return -EOPNOTSUPP;
4826 if (sa->sa_family != dev->type)
4827 return -EINVAL;
4828 if (!netif_device_present(dev))
4829 return -ENODEV;
4830 err = ops->ndo_set_mac_address(dev, sa);
4831 if (err)
4832 return err;
4833 dev->addr_assign_type = NET_ADDR_SET;
4834 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4835 add_device_randomness(dev->dev_addr, dev->addr_len);
4836 return 0;
4837 }
4838 EXPORT_SYMBOL(dev_set_mac_address);
4839
4840 /**
4841 * dev_change_carrier - Change device carrier
4842 * @dev: device
4843 * @new_carrier: new value
4844 *
4845 * Change device carrier
4846 */
dev_change_carrier(struct net_device * dev,bool new_carrier)4847 int dev_change_carrier(struct net_device *dev, bool new_carrier)
4848 {
4849 const struct net_device_ops *ops = dev->netdev_ops;
4850
4851 if (!ops->ndo_change_carrier)
4852 return -EOPNOTSUPP;
4853 if (!netif_device_present(dev))
4854 return -ENODEV;
4855 return ops->ndo_change_carrier(dev, new_carrier);
4856 }
4857 EXPORT_SYMBOL(dev_change_carrier);
4858
4859 /**
4860 * dev_new_index - allocate an ifindex
4861 * @net: the applicable net namespace
4862 *
4863 * Returns a suitable unique value for a new device interface
4864 * number. The caller must hold the rtnl semaphore or the
4865 * dev_base_lock to be sure it remains unique.
4866 */
dev_new_index(struct net * net)4867 static int dev_new_index(struct net *net)
4868 {
4869 int ifindex = net->ifindex;
4870 for (;;) {
4871 if (++ifindex <= 0)
4872 ifindex = 1;
4873 if (!__dev_get_by_index(net, ifindex))
4874 return net->ifindex = ifindex;
4875 }
4876 }
4877
4878 /* Delayed registration/unregisteration */
4879 static LIST_HEAD(net_todo_list);
4880
net_set_todo(struct net_device * dev)4881 static void net_set_todo(struct net_device *dev)
4882 {
4883 list_add_tail(&dev->todo_list, &net_todo_list);
4884 }
4885
rollback_registered_many(struct list_head * head)4886 static void rollback_registered_many(struct list_head *head)
4887 {
4888 struct net_device *dev, *tmp;
4889
4890 BUG_ON(dev_boot_phase);
4891 ASSERT_RTNL();
4892
4893 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
4894 /* Some devices call without registering
4895 * for initialization unwind. Remove those
4896 * devices and proceed with the remaining.
4897 */
4898 if (dev->reg_state == NETREG_UNINITIALIZED) {
4899 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
4900 dev->name, dev);
4901
4902 WARN_ON(1);
4903 list_del(&dev->unreg_list);
4904 continue;
4905 }
4906 dev->dismantle = true;
4907 BUG_ON(dev->reg_state != NETREG_REGISTERED);
4908 }
4909
4910 /* If device is running, close it first. */
4911 dev_close_many(head);
4912
4913 list_for_each_entry(dev, head, unreg_list) {
4914 /* And unlink it from device chain. */
4915 unlist_netdevice(dev);
4916
4917 dev->reg_state = NETREG_UNREGISTERING;
4918 }
4919
4920 synchronize_net();
4921
4922 list_for_each_entry(dev, head, unreg_list) {
4923 /* Shutdown queueing discipline. */
4924 dev_shutdown(dev);
4925
4926
4927 /* Notify protocols, that we are about to destroy
4928 this device. They should clean all the things.
4929 */
4930 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4931
4932 if (!dev->rtnl_link_ops ||
4933 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4934 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
4935
4936 /*
4937 * Flush the unicast and multicast chains
4938 */
4939 dev_uc_flush(dev);
4940 dev_mc_flush(dev);
4941
4942 if (dev->netdev_ops->ndo_uninit)
4943 dev->netdev_ops->ndo_uninit(dev);
4944
4945 /* Notifier chain MUST detach us all upper devices. */
4946 WARN_ON(netdev_has_any_upper_dev(dev));
4947
4948 /* Remove entries from kobject tree */
4949 netdev_unregister_kobject(dev);
4950 #ifdef CONFIG_XPS
4951 /* Remove XPS queueing entries */
4952 netif_reset_xps_queues_gt(dev, 0);
4953 #endif
4954 }
4955
4956 synchronize_net();
4957
4958 list_for_each_entry(dev, head, unreg_list)
4959 dev_put(dev);
4960 }
4961
rollback_registered(struct net_device * dev)4962 static void rollback_registered(struct net_device *dev)
4963 {
4964 LIST_HEAD(single);
4965
4966 list_add(&dev->unreg_list, &single);
4967 rollback_registered_many(&single);
4968 list_del(&single);
4969 }
4970
netdev_fix_features(struct net_device * dev,netdev_features_t features)4971 static netdev_features_t netdev_fix_features(struct net_device *dev,
4972 netdev_features_t features)
4973 {
4974 /* Fix illegal checksum combinations */
4975 if ((features & NETIF_F_HW_CSUM) &&
4976 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4977 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
4978 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4979 }
4980
4981 /* TSO requires that SG is present as well. */
4982 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
4983 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
4984 features &= ~NETIF_F_ALL_TSO;
4985 }
4986
4987 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
4988 !(features & NETIF_F_IP_CSUM)) {
4989 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
4990 features &= ~NETIF_F_TSO;
4991 features &= ~NETIF_F_TSO_ECN;
4992 }
4993
4994 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
4995 !(features & NETIF_F_IPV6_CSUM)) {
4996 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
4997 features &= ~NETIF_F_TSO6;
4998 }
4999
5000 /* TSO ECN requires that TSO is present as well. */
5001 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5002 features &= ~NETIF_F_TSO_ECN;
5003
5004 /* Software GSO depends on SG. */
5005 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
5006 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
5007 features &= ~NETIF_F_GSO;
5008 }
5009
5010 /* UFO needs SG and checksumming */
5011 if (features & NETIF_F_UFO) {
5012 /* maybe split UFO into V4 and V6? */
5013 if (!((features & NETIF_F_GEN_CSUM) ||
5014 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5015 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5016 netdev_dbg(dev,
5017 "Dropping NETIF_F_UFO since no checksum offload features.\n");
5018 features &= ~NETIF_F_UFO;
5019 }
5020
5021 if (!(features & NETIF_F_SG)) {
5022 netdev_dbg(dev,
5023 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
5024 features &= ~NETIF_F_UFO;
5025 }
5026 }
5027
5028 return features;
5029 }
5030
__netdev_update_features(struct net_device * dev)5031 int __netdev_update_features(struct net_device *dev)
5032 {
5033 netdev_features_t features;
5034 int err = 0;
5035
5036 ASSERT_RTNL();
5037
5038 features = netdev_get_wanted_features(dev);
5039
5040 if (dev->netdev_ops->ndo_fix_features)
5041 features = dev->netdev_ops->ndo_fix_features(dev, features);
5042
5043 /* driver might be less strict about feature dependencies */
5044 features = netdev_fix_features(dev, features);
5045
5046 if (dev->features == features)
5047 return 0;
5048
5049 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5050 &dev->features, &features);
5051
5052 if (dev->netdev_ops->ndo_set_features)
5053 err = dev->netdev_ops->ndo_set_features(dev, features);
5054
5055 if (unlikely(err < 0)) {
5056 netdev_err(dev,
5057 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5058 err, &features, &dev->features);
5059 return -1;
5060 }
5061
5062 if (!err)
5063 dev->features = features;
5064
5065 return 1;
5066 }
5067
5068 /**
5069 * netdev_update_features - recalculate device features
5070 * @dev: the device to check
5071 *
5072 * Recalculate dev->features set and send notifications if it
5073 * has changed. Should be called after driver or hardware dependent
5074 * conditions might have changed that influence the features.
5075 */
netdev_update_features(struct net_device * dev)5076 void netdev_update_features(struct net_device *dev)
5077 {
5078 if (__netdev_update_features(dev))
5079 netdev_features_change(dev);
5080 }
5081 EXPORT_SYMBOL(netdev_update_features);
5082
5083 /**
5084 * netdev_change_features - recalculate device features
5085 * @dev: the device to check
5086 *
5087 * Recalculate dev->features set and send notifications even
5088 * if they have not changed. Should be called instead of
5089 * netdev_update_features() if also dev->vlan_features might
5090 * have changed to allow the changes to be propagated to stacked
5091 * VLAN devices.
5092 */
netdev_change_features(struct net_device * dev)5093 void netdev_change_features(struct net_device *dev)
5094 {
5095 __netdev_update_features(dev);
5096 netdev_features_change(dev);
5097 }
5098 EXPORT_SYMBOL(netdev_change_features);
5099
5100 /**
5101 * netif_stacked_transfer_operstate - transfer operstate
5102 * @rootdev: the root or lower level device to transfer state from
5103 * @dev: the device to transfer operstate to
5104 *
5105 * Transfer operational state from root to device. This is normally
5106 * called when a stacking relationship exists between the root
5107 * device and the device(a leaf device).
5108 */
netif_stacked_transfer_operstate(const struct net_device * rootdev,struct net_device * dev)5109 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5110 struct net_device *dev)
5111 {
5112 if (rootdev->operstate == IF_OPER_DORMANT)
5113 netif_dormant_on(dev);
5114 else
5115 netif_dormant_off(dev);
5116
5117 if (netif_carrier_ok(rootdev)) {
5118 if (!netif_carrier_ok(dev))
5119 netif_carrier_on(dev);
5120 } else {
5121 if (netif_carrier_ok(dev))
5122 netif_carrier_off(dev);
5123 }
5124 }
5125 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5126
5127 #ifdef CONFIG_RPS
netif_alloc_rx_queues(struct net_device * dev)5128 static int netif_alloc_rx_queues(struct net_device *dev)
5129 {
5130 unsigned int i, count = dev->num_rx_queues;
5131 struct netdev_rx_queue *rx;
5132
5133 BUG_ON(count < 1);
5134
5135 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5136 if (!rx)
5137 return -ENOMEM;
5138
5139 dev->_rx = rx;
5140
5141 for (i = 0; i < count; i++)
5142 rx[i].dev = dev;
5143 return 0;
5144 }
5145 #endif
5146
netdev_init_one_queue(struct net_device * dev,struct netdev_queue * queue,void * _unused)5147 static void netdev_init_one_queue(struct net_device *dev,
5148 struct netdev_queue *queue, void *_unused)
5149 {
5150 /* Initialize queue lock */
5151 spin_lock_init(&queue->_xmit_lock);
5152 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5153 queue->xmit_lock_owner = -1;
5154 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5155 queue->dev = dev;
5156 #ifdef CONFIG_BQL
5157 dql_init(&queue->dql, HZ);
5158 #endif
5159 }
5160
netif_alloc_netdev_queues(struct net_device * dev)5161 static int netif_alloc_netdev_queues(struct net_device *dev)
5162 {
5163 unsigned int count = dev->num_tx_queues;
5164 struct netdev_queue *tx;
5165
5166 BUG_ON(count < 1);
5167
5168 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5169 if (!tx)
5170 return -ENOMEM;
5171
5172 dev->_tx = tx;
5173
5174 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5175 spin_lock_init(&dev->tx_global_lock);
5176
5177 return 0;
5178 }
5179
5180 /**
5181 * register_netdevice - register a network device
5182 * @dev: device to register
5183 *
5184 * Take a completed network device structure and add it to the kernel
5185 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5186 * chain. 0 is returned on success. A negative errno code is returned
5187 * on a failure to set up the device, or if the name is a duplicate.
5188 *
5189 * Callers must hold the rtnl semaphore. You may want
5190 * register_netdev() instead of this.
5191 *
5192 * BUGS:
5193 * The locking appears insufficient to guarantee two parallel registers
5194 * will not get the same name.
5195 */
5196
register_netdevice(struct net_device * dev)5197 int register_netdevice(struct net_device *dev)
5198 {
5199 int ret;
5200 struct net *net = dev_net(dev);
5201
5202 BUG_ON(dev_boot_phase);
5203 ASSERT_RTNL();
5204
5205 might_sleep();
5206
5207 /* When net_device's are persistent, this will be fatal. */
5208 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5209 BUG_ON(!net);
5210
5211 spin_lock_init(&dev->addr_list_lock);
5212 netdev_set_addr_lockdep_class(dev);
5213
5214 dev->iflink = -1;
5215
5216 ret = dev_get_valid_name(net, dev, dev->name);
5217 if (ret < 0)
5218 goto out;
5219
5220 /* Init, if this function is available */
5221 if (dev->netdev_ops->ndo_init) {
5222 ret = dev->netdev_ops->ndo_init(dev);
5223 if (ret) {
5224 if (ret > 0)
5225 ret = -EIO;
5226 goto out;
5227 }
5228 }
5229
5230 if (((dev->hw_features | dev->features) &
5231 NETIF_F_HW_VLAN_CTAG_FILTER) &&
5232 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
5233 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
5234 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
5235 ret = -EINVAL;
5236 goto err_uninit;
5237 }
5238
5239 ret = -EBUSY;
5240 if (!dev->ifindex)
5241 dev->ifindex = dev_new_index(net);
5242 else if (__dev_get_by_index(net, dev->ifindex))
5243 goto err_uninit;
5244
5245 if (dev->iflink == -1)
5246 dev->iflink = dev->ifindex;
5247
5248 /* Transfer changeable features to wanted_features and enable
5249 * software offloads (GSO and GRO).
5250 */
5251 dev->hw_features |= NETIF_F_SOFT_FEATURES;
5252 dev->features |= NETIF_F_SOFT_FEATURES;
5253 dev->wanted_features = dev->features & dev->hw_features;
5254
5255 /* Turn on no cache copy if HW is doing checksum */
5256 if (!(dev->flags & IFF_LOOPBACK)) {
5257 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5258 if (dev->features & NETIF_F_ALL_CSUM) {
5259 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5260 dev->features |= NETIF_F_NOCACHE_COPY;
5261 }
5262 }
5263
5264 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
5265 */
5266 dev->vlan_features |= NETIF_F_HIGHDMA;
5267
5268 /* Make NETIF_F_SG inheritable to tunnel devices.
5269 */
5270 dev->hw_enc_features |= NETIF_F_SG;
5271
5272 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5273 ret = notifier_to_errno(ret);
5274 if (ret)
5275 goto err_uninit;
5276
5277 ret = netdev_register_kobject(dev);
5278 if (ret)
5279 goto err_uninit;
5280 dev->reg_state = NETREG_REGISTERED;
5281
5282 __netdev_update_features(dev);
5283
5284 /*
5285 * Default initial state at registry is that the
5286 * device is present.
5287 */
5288
5289 set_bit(__LINK_STATE_PRESENT, &dev->state);
5290
5291 linkwatch_init_dev(dev);
5292
5293 dev_init_scheduler(dev);
5294 dev_hold(dev);
5295 list_netdevice(dev);
5296 add_device_randomness(dev->dev_addr, dev->addr_len);
5297
5298 /* If the device has permanent device address, driver should
5299 * set dev_addr and also addr_assign_type should be set to
5300 * NET_ADDR_PERM (default value).
5301 */
5302 if (dev->addr_assign_type == NET_ADDR_PERM)
5303 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
5304
5305 /* Notify protocols, that a new device appeared. */
5306 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5307 ret = notifier_to_errno(ret);
5308 if (ret) {
5309 rollback_registered(dev);
5310 dev->reg_state = NETREG_UNREGISTERED;
5311 }
5312 /*
5313 * Prevent userspace races by waiting until the network
5314 * device is fully setup before sending notifications.
5315 */
5316 if (!dev->rtnl_link_ops ||
5317 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5318 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5319
5320 out:
5321 return ret;
5322
5323 err_uninit:
5324 if (dev->netdev_ops->ndo_uninit)
5325 dev->netdev_ops->ndo_uninit(dev);
5326 goto out;
5327 }
5328 EXPORT_SYMBOL(register_netdevice);
5329
5330 /**
5331 * init_dummy_netdev - init a dummy network device for NAPI
5332 * @dev: device to init
5333 *
5334 * This takes a network device structure and initialize the minimum
5335 * amount of fields so it can be used to schedule NAPI polls without
5336 * registering a full blown interface. This is to be used by drivers
5337 * that need to tie several hardware interfaces to a single NAPI
5338 * poll scheduler due to HW limitations.
5339 */
init_dummy_netdev(struct net_device * dev)5340 int init_dummy_netdev(struct net_device *dev)
5341 {
5342 /* Clear everything. Note we don't initialize spinlocks
5343 * are they aren't supposed to be taken by any of the
5344 * NAPI code and this dummy netdev is supposed to be
5345 * only ever used for NAPI polls
5346 */
5347 memset(dev, 0, sizeof(struct net_device));
5348
5349 /* make sure we BUG if trying to hit standard
5350 * register/unregister code path
5351 */
5352 dev->reg_state = NETREG_DUMMY;
5353
5354 /* NAPI wants this */
5355 INIT_LIST_HEAD(&dev->napi_list);
5356
5357 /* a dummy interface is started by default */
5358 set_bit(__LINK_STATE_PRESENT, &dev->state);
5359 set_bit(__LINK_STATE_START, &dev->state);
5360
5361 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5362 * because users of this 'device' dont need to change
5363 * its refcount.
5364 */
5365
5366 return 0;
5367 }
5368 EXPORT_SYMBOL_GPL(init_dummy_netdev);
5369
5370
5371 /**
5372 * register_netdev - register a network device
5373 * @dev: device to register
5374 *
5375 * Take a completed network device structure and add it to the kernel
5376 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5377 * chain. 0 is returned on success. A negative errno code is returned
5378 * on a failure to set up the device, or if the name is a duplicate.
5379 *
5380 * This is a wrapper around register_netdevice that takes the rtnl semaphore
5381 * and expands the device name if you passed a format string to
5382 * alloc_netdev.
5383 */
register_netdev(struct net_device * dev)5384 int register_netdev(struct net_device *dev)
5385 {
5386 int err;
5387
5388 rtnl_lock();
5389 err = register_netdevice(dev);
5390 rtnl_unlock();
5391 return err;
5392 }
5393 EXPORT_SYMBOL(register_netdev);
5394
netdev_refcnt_read(const struct net_device * dev)5395 int netdev_refcnt_read(const struct net_device *dev)
5396 {
5397 int i, refcnt = 0;
5398
5399 for_each_possible_cpu(i)
5400 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5401 return refcnt;
5402 }
5403 EXPORT_SYMBOL(netdev_refcnt_read);
5404
5405 /**
5406 * netdev_wait_allrefs - wait until all references are gone.
5407 * @dev: target net_device
5408 *
5409 * This is called when unregistering network devices.
5410 *
5411 * Any protocol or device that holds a reference should register
5412 * for netdevice notification, and cleanup and put back the
5413 * reference if they receive an UNREGISTER event.
5414 * We can get stuck here if buggy protocols don't correctly
5415 * call dev_put.
5416 */
netdev_wait_allrefs(struct net_device * dev)5417 static void netdev_wait_allrefs(struct net_device *dev)
5418 {
5419 unsigned long rebroadcast_time, warning_time;
5420 int refcnt;
5421
5422 linkwatch_forget_dev(dev);
5423
5424 rebroadcast_time = warning_time = jiffies;
5425 refcnt = netdev_refcnt_read(dev);
5426
5427 while (refcnt != 0) {
5428 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5429 rtnl_lock();
5430
5431 /* Rebroadcast unregister notification */
5432 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5433
5434 __rtnl_unlock();
5435 rcu_barrier();
5436 rtnl_lock();
5437
5438 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5439 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5440 &dev->state)) {
5441 /* We must not have linkwatch events
5442 * pending on unregister. If this
5443 * happens, we simply run the queue
5444 * unscheduled, resulting in a noop
5445 * for this device.
5446 */
5447 linkwatch_run_queue();
5448 }
5449
5450 __rtnl_unlock();
5451
5452 rebroadcast_time = jiffies;
5453 }
5454
5455 msleep(250);
5456
5457 refcnt = netdev_refcnt_read(dev);
5458
5459 if (time_after(jiffies, warning_time + 10 * HZ)) {
5460 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5461 dev->name, refcnt);
5462 warning_time = jiffies;
5463 }
5464 }
5465 }
5466
5467 /* The sequence is:
5468 *
5469 * rtnl_lock();
5470 * ...
5471 * register_netdevice(x1);
5472 * register_netdevice(x2);
5473 * ...
5474 * unregister_netdevice(y1);
5475 * unregister_netdevice(y2);
5476 * ...
5477 * rtnl_unlock();
5478 * free_netdev(y1);
5479 * free_netdev(y2);
5480 *
5481 * We are invoked by rtnl_unlock().
5482 * This allows us to deal with problems:
5483 * 1) We can delete sysfs objects which invoke hotplug
5484 * without deadlocking with linkwatch via keventd.
5485 * 2) Since we run with the RTNL semaphore not held, we can sleep
5486 * safely in order to wait for the netdev refcnt to drop to zero.
5487 *
5488 * We must not return until all unregister events added during
5489 * the interval the lock was held have been completed.
5490 */
netdev_run_todo(void)5491 void netdev_run_todo(void)
5492 {
5493 struct list_head list;
5494
5495 /* Snapshot list, allow later requests */
5496 list_replace_init(&net_todo_list, &list);
5497
5498 __rtnl_unlock();
5499
5500
5501 /* Wait for rcu callbacks to finish before next phase */
5502 if (!list_empty(&list))
5503 rcu_barrier();
5504
5505 while (!list_empty(&list)) {
5506 struct net_device *dev
5507 = list_first_entry(&list, struct net_device, todo_list);
5508 list_del(&dev->todo_list);
5509
5510 rtnl_lock();
5511 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5512 __rtnl_unlock();
5513
5514 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5515 pr_err("network todo '%s' but state %d\n",
5516 dev->name, dev->reg_state);
5517 dump_stack();
5518 continue;
5519 }
5520
5521 dev->reg_state = NETREG_UNREGISTERED;
5522
5523 on_each_cpu(flush_backlog, dev, 1);
5524
5525 netdev_wait_allrefs(dev);
5526
5527 /* paranoia */
5528 BUG_ON(netdev_refcnt_read(dev));
5529 WARN_ON(rcu_access_pointer(dev->ip_ptr));
5530 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
5531 WARN_ON(dev->dn_ptr);
5532
5533 if (dev->destructor)
5534 dev->destructor(dev);
5535
5536 /* Free network device */
5537 kobject_put(&dev->dev.kobj);
5538 }
5539 }
5540
5541 /* Convert net_device_stats to rtnl_link_stats64. They have the same
5542 * fields in the same order, with only the type differing.
5543 */
netdev_stats_to_stats64(struct rtnl_link_stats64 * stats64,const struct net_device_stats * netdev_stats)5544 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5545 const struct net_device_stats *netdev_stats)
5546 {
5547 #if BITS_PER_LONG == 64
5548 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5549 memcpy(stats64, netdev_stats, sizeof(*stats64));
5550 #else
5551 size_t i, n = sizeof(*stats64) / sizeof(u64);
5552 const unsigned long *src = (const unsigned long *)netdev_stats;
5553 u64 *dst = (u64 *)stats64;
5554
5555 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5556 sizeof(*stats64) / sizeof(u64));
5557 for (i = 0; i < n; i++)
5558 dst[i] = src[i];
5559 #endif
5560 }
5561 EXPORT_SYMBOL(netdev_stats_to_stats64);
5562
5563 /**
5564 * dev_get_stats - get network device statistics
5565 * @dev: device to get statistics from
5566 * @storage: place to store stats
5567 *
5568 * Get network statistics from device. Return @storage.
5569 * The device driver may provide its own method by setting
5570 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5571 * otherwise the internal statistics structure is used.
5572 */
dev_get_stats(struct net_device * dev,struct rtnl_link_stats64 * storage)5573 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5574 struct rtnl_link_stats64 *storage)
5575 {
5576 const struct net_device_ops *ops = dev->netdev_ops;
5577
5578 if (ops->ndo_get_stats64) {
5579 memset(storage, 0, sizeof(*storage));
5580 ops->ndo_get_stats64(dev, storage);
5581 } else if (ops->ndo_get_stats) {
5582 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5583 } else {
5584 netdev_stats_to_stats64(storage, &dev->stats);
5585 }
5586 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5587 return storage;
5588 }
5589 EXPORT_SYMBOL(dev_get_stats);
5590
dev_ingress_queue_create(struct net_device * dev)5591 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5592 {
5593 struct netdev_queue *queue = dev_ingress_queue(dev);
5594
5595 #ifdef CONFIG_NET_CLS_ACT
5596 if (queue)
5597 return queue;
5598 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5599 if (!queue)
5600 return NULL;
5601 netdev_init_one_queue(dev, queue, NULL);
5602 queue->qdisc = &noop_qdisc;
5603 queue->qdisc_sleeping = &noop_qdisc;
5604 rcu_assign_pointer(dev->ingress_queue, queue);
5605 #endif
5606 return queue;
5607 }
5608
5609 static const struct ethtool_ops default_ethtool_ops;
5610
netdev_set_default_ethtool_ops(struct net_device * dev,const struct ethtool_ops * ops)5611 void netdev_set_default_ethtool_ops(struct net_device *dev,
5612 const struct ethtool_ops *ops)
5613 {
5614 if (dev->ethtool_ops == &default_ethtool_ops)
5615 dev->ethtool_ops = ops;
5616 }
5617 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
5618
5619 /**
5620 * alloc_netdev_mqs - allocate network device
5621 * @sizeof_priv: size of private data to allocate space for
5622 * @name: device name format string
5623 * @setup: callback to initialize device
5624 * @txqs: the number of TX subqueues to allocate
5625 * @rxqs: the number of RX subqueues to allocate
5626 *
5627 * Allocates a struct net_device with private data area for driver use
5628 * and performs basic initialization. Also allocates subquue structs
5629 * for each queue on the device.
5630 */
alloc_netdev_mqs(int sizeof_priv,const char * name,void (* setup)(struct net_device *),unsigned int txqs,unsigned int rxqs)5631 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5632 void (*setup)(struct net_device *),
5633 unsigned int txqs, unsigned int rxqs)
5634 {
5635 struct net_device *dev;
5636 size_t alloc_size;
5637 struct net_device *p;
5638
5639 BUG_ON(strlen(name) >= sizeof(dev->name));
5640
5641 if (txqs < 1) {
5642 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
5643 return NULL;
5644 }
5645
5646 #ifdef CONFIG_RPS
5647 if (rxqs < 1) {
5648 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
5649 return NULL;
5650 }
5651 #endif
5652
5653 alloc_size = sizeof(struct net_device);
5654 if (sizeof_priv) {
5655 /* ensure 32-byte alignment of private area */
5656 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5657 alloc_size += sizeof_priv;
5658 }
5659 /* ensure 32-byte alignment of whole construct */
5660 alloc_size += NETDEV_ALIGN - 1;
5661
5662 p = kzalloc(alloc_size, GFP_KERNEL);
5663 if (!p)
5664 return NULL;
5665
5666 dev = PTR_ALIGN(p, NETDEV_ALIGN);
5667 dev->padded = (char *)dev - (char *)p;
5668
5669 dev->pcpu_refcnt = alloc_percpu(int);
5670 if (!dev->pcpu_refcnt)
5671 goto free_p;
5672
5673 if (dev_addr_init(dev))
5674 goto free_pcpu;
5675
5676 dev_mc_init(dev);
5677 dev_uc_init(dev);
5678
5679 dev_net_set(dev, &init_net);
5680
5681 dev->gso_max_size = GSO_MAX_SIZE;
5682 dev->gso_max_segs = GSO_MAX_SEGS;
5683
5684 INIT_LIST_HEAD(&dev->napi_list);
5685 INIT_LIST_HEAD(&dev->unreg_list);
5686 INIT_LIST_HEAD(&dev->link_watch_list);
5687 INIT_LIST_HEAD(&dev->upper_dev_list);
5688 dev->priv_flags = IFF_XMIT_DST_RELEASE;
5689 setup(dev);
5690
5691 dev->num_tx_queues = txqs;
5692 dev->real_num_tx_queues = txqs;
5693 if (netif_alloc_netdev_queues(dev))
5694 goto free_all;
5695
5696 #ifdef CONFIG_RPS
5697 dev->num_rx_queues = rxqs;
5698 dev->real_num_rx_queues = rxqs;
5699 if (netif_alloc_rx_queues(dev))
5700 goto free_all;
5701 #endif
5702
5703 strcpy(dev->name, name);
5704 dev->group = INIT_NETDEV_GROUP;
5705 if (!dev->ethtool_ops)
5706 dev->ethtool_ops = &default_ethtool_ops;
5707 return dev;
5708
5709 free_all:
5710 free_netdev(dev);
5711 return NULL;
5712
5713 free_pcpu:
5714 free_percpu(dev->pcpu_refcnt);
5715 kfree(dev->_tx);
5716 #ifdef CONFIG_RPS
5717 kfree(dev->_rx);
5718 #endif
5719
5720 free_p:
5721 kfree(p);
5722 return NULL;
5723 }
5724 EXPORT_SYMBOL(alloc_netdev_mqs);
5725
5726 /**
5727 * free_netdev - free network device
5728 * @dev: device
5729 *
5730 * This function does the last stage of destroying an allocated device
5731 * interface. The reference to the device object is released.
5732 * If this is the last reference then it will be freed.
5733 */
free_netdev(struct net_device * dev)5734 void free_netdev(struct net_device *dev)
5735 {
5736 struct napi_struct *p, *n;
5737
5738 release_net(dev_net(dev));
5739
5740 kfree(dev->_tx);
5741 #ifdef CONFIG_RPS
5742 kfree(dev->_rx);
5743 #endif
5744
5745 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
5746
5747 /* Flush device addresses */
5748 dev_addr_flush(dev);
5749
5750 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5751 netif_napi_del(p);
5752
5753 free_percpu(dev->pcpu_refcnt);
5754 dev->pcpu_refcnt = NULL;
5755
5756 /* Compatibility with error handling in drivers */
5757 if (dev->reg_state == NETREG_UNINITIALIZED) {
5758 kfree((char *)dev - dev->padded);
5759 return;
5760 }
5761
5762 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5763 dev->reg_state = NETREG_RELEASED;
5764
5765 /* will free via device release */
5766 put_device(&dev->dev);
5767 }
5768 EXPORT_SYMBOL(free_netdev);
5769
5770 /**
5771 * synchronize_net - Synchronize with packet receive processing
5772 *
5773 * Wait for packets currently being received to be done.
5774 * Does not block later packets from starting.
5775 */
synchronize_net(void)5776 void synchronize_net(void)
5777 {
5778 might_sleep();
5779 if (rtnl_is_locked())
5780 synchronize_rcu_expedited();
5781 else
5782 synchronize_rcu();
5783 }
5784 EXPORT_SYMBOL(synchronize_net);
5785
5786 /**
5787 * unregister_netdevice_queue - remove device from the kernel
5788 * @dev: device
5789 * @head: list
5790 *
5791 * This function shuts down a device interface and removes it
5792 * from the kernel tables.
5793 * If head not NULL, device is queued to be unregistered later.
5794 *
5795 * Callers must hold the rtnl semaphore. You may want
5796 * unregister_netdev() instead of this.
5797 */
5798
unregister_netdevice_queue(struct net_device * dev,struct list_head * head)5799 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5800 {
5801 ASSERT_RTNL();
5802
5803 if (head) {
5804 list_move_tail(&dev->unreg_list, head);
5805 } else {
5806 rollback_registered(dev);
5807 /* Finish processing unregister after unlock */
5808 net_set_todo(dev);
5809 }
5810 }
5811 EXPORT_SYMBOL(unregister_netdevice_queue);
5812
5813 /**
5814 * unregister_netdevice_many - unregister many devices
5815 * @head: list of devices
5816 */
unregister_netdevice_many(struct list_head * head)5817 void unregister_netdevice_many(struct list_head *head)
5818 {
5819 struct net_device *dev;
5820
5821 if (!list_empty(head)) {
5822 rollback_registered_many(head);
5823 list_for_each_entry(dev, head, unreg_list)
5824 net_set_todo(dev);
5825 }
5826 }
5827 EXPORT_SYMBOL(unregister_netdevice_many);
5828
5829 /**
5830 * unregister_netdev - remove device from the kernel
5831 * @dev: device
5832 *
5833 * This function shuts down a device interface and removes it
5834 * from the kernel tables.
5835 *
5836 * This is just a wrapper for unregister_netdevice that takes
5837 * the rtnl semaphore. In general you want to use this and not
5838 * unregister_netdevice.
5839 */
unregister_netdev(struct net_device * dev)5840 void unregister_netdev(struct net_device *dev)
5841 {
5842 rtnl_lock();
5843 unregister_netdevice(dev);
5844 rtnl_unlock();
5845 }
5846 EXPORT_SYMBOL(unregister_netdev);
5847
5848 /**
5849 * dev_change_net_namespace - move device to different nethost namespace
5850 * @dev: device
5851 * @net: network namespace
5852 * @pat: If not NULL name pattern to try if the current device name
5853 * is already taken in the destination network namespace.
5854 *
5855 * This function shuts down a device interface and moves it
5856 * to a new network namespace. On success 0 is returned, on
5857 * a failure a netagive errno code is returned.
5858 *
5859 * Callers must hold the rtnl semaphore.
5860 */
5861
dev_change_net_namespace(struct net_device * dev,struct net * net,const char * pat)5862 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5863 {
5864 int err;
5865
5866 ASSERT_RTNL();
5867
5868 /* Don't allow namespace local devices to be moved. */
5869 err = -EINVAL;
5870 if (dev->features & NETIF_F_NETNS_LOCAL)
5871 goto out;
5872
5873 /* Ensure the device has been registrered */
5874 if (dev->reg_state != NETREG_REGISTERED)
5875 goto out;
5876
5877 /* Get out if there is nothing todo */
5878 err = 0;
5879 if (net_eq(dev_net(dev), net))
5880 goto out;
5881
5882 /* Pick the destination device name, and ensure
5883 * we can use it in the destination network namespace.
5884 */
5885 err = -EEXIST;
5886 if (__dev_get_by_name(net, dev->name)) {
5887 /* We get here if we can't use the current device name */
5888 if (!pat)
5889 goto out;
5890 if (dev_get_valid_name(net, dev, pat) < 0)
5891 goto out;
5892 }
5893
5894 /*
5895 * And now a mini version of register_netdevice unregister_netdevice.
5896 */
5897
5898 /* If device is running close it first. */
5899 dev_close(dev);
5900
5901 /* And unlink it from device chain */
5902 err = -ENODEV;
5903 unlist_netdevice(dev);
5904
5905 synchronize_net();
5906
5907 /* Shutdown queueing discipline. */
5908 dev_shutdown(dev);
5909
5910 /* Notify protocols, that we are about to destroy
5911 this device. They should clean all the things.
5912
5913 Note that dev->reg_state stays at NETREG_REGISTERED.
5914 This is wanted because this way 8021q and macvlan know
5915 the device is just moving and can keep their slaves up.
5916 */
5917 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5918 rcu_barrier();
5919 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5920 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5921
5922 /*
5923 * Flush the unicast and multicast chains
5924 */
5925 dev_uc_flush(dev);
5926 dev_mc_flush(dev);
5927
5928 /* Send a netdev-removed uevent to the old namespace */
5929 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
5930
5931 /* Actually switch the network namespace */
5932 dev_net_set(dev, net);
5933
5934 /* If there is an ifindex conflict assign a new one */
5935 if (__dev_get_by_index(net, dev->ifindex)) {
5936 int iflink = (dev->iflink == dev->ifindex);
5937 dev->ifindex = dev_new_index(net);
5938 if (iflink)
5939 dev->iflink = dev->ifindex;
5940 }
5941
5942 /* Send a netdev-add uevent to the new namespace */
5943 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
5944
5945 /* Fixup kobjects */
5946 err = device_rename(&dev->dev, dev->name);
5947 WARN_ON(err);
5948
5949 /* Add the device back in the hashes */
5950 list_netdevice(dev);
5951
5952 /* Notify protocols, that a new device appeared. */
5953 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5954
5955 /*
5956 * Prevent userspace races by waiting until the network
5957 * device is fully setup before sending notifications.
5958 */
5959 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5960
5961 synchronize_net();
5962 err = 0;
5963 out:
5964 return err;
5965 }
5966 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5967
dev_cpu_callback(struct notifier_block * nfb,unsigned long action,void * ocpu)5968 static int dev_cpu_callback(struct notifier_block *nfb,
5969 unsigned long action,
5970 void *ocpu)
5971 {
5972 struct sk_buff **list_skb;
5973 struct sk_buff *skb;
5974 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5975 struct softnet_data *sd, *oldsd;
5976
5977 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
5978 return NOTIFY_OK;
5979
5980 local_irq_disable();
5981 cpu = smp_processor_id();
5982 sd = &per_cpu(softnet_data, cpu);
5983 oldsd = &per_cpu(softnet_data, oldcpu);
5984
5985 /* Find end of our completion_queue. */
5986 list_skb = &sd->completion_queue;
5987 while (*list_skb)
5988 list_skb = &(*list_skb)->next;
5989 /* Append completion queue from offline CPU. */
5990 *list_skb = oldsd->completion_queue;
5991 oldsd->completion_queue = NULL;
5992
5993 /* Append output queue from offline CPU. */
5994 if (oldsd->output_queue) {
5995 *sd->output_queue_tailp = oldsd->output_queue;
5996 sd->output_queue_tailp = oldsd->output_queue_tailp;
5997 oldsd->output_queue = NULL;
5998 oldsd->output_queue_tailp = &oldsd->output_queue;
5999 }
6000 /* Append NAPI poll list from offline CPU. */
6001 if (!list_empty(&oldsd->poll_list)) {
6002 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6003 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6004 }
6005
6006 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6007 local_irq_enable();
6008
6009 /* Process offline CPU's input_pkt_queue */
6010 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6011 netif_rx(skb);
6012 input_queue_head_incr(oldsd);
6013 }
6014 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
6015 netif_rx(skb);
6016 input_queue_head_incr(oldsd);
6017 }
6018
6019 return NOTIFY_OK;
6020 }
6021
6022
6023 /**
6024 * netdev_increment_features - increment feature set by one
6025 * @all: current feature set
6026 * @one: new feature set
6027 * @mask: mask feature set
6028 *
6029 * Computes a new feature set after adding a device with feature set
6030 * @one to the master device with current feature set @all. Will not
6031 * enable anything that is off in @mask. Returns the new feature set.
6032 */
netdev_increment_features(netdev_features_t all,netdev_features_t one,netdev_features_t mask)6033 netdev_features_t netdev_increment_features(netdev_features_t all,
6034 netdev_features_t one, netdev_features_t mask)
6035 {
6036 if (mask & NETIF_F_GEN_CSUM)
6037 mask |= NETIF_F_ALL_CSUM;
6038 mask |= NETIF_F_VLAN_CHALLENGED;
6039
6040 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6041 all &= one | ~NETIF_F_ALL_FOR_ALL;
6042
6043 /* If one device supports hw checksumming, set for all. */
6044 if (all & NETIF_F_GEN_CSUM)
6045 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
6046
6047 return all;
6048 }
6049 EXPORT_SYMBOL(netdev_increment_features);
6050
netdev_create_hash(void)6051 static struct hlist_head *netdev_create_hash(void)
6052 {
6053 int i;
6054 struct hlist_head *hash;
6055
6056 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6057 if (hash != NULL)
6058 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6059 INIT_HLIST_HEAD(&hash[i]);
6060
6061 return hash;
6062 }
6063
6064 /* Initialize per network namespace state */
netdev_init(struct net * net)6065 static int __net_init netdev_init(struct net *net)
6066 {
6067 if (net != &init_net)
6068 INIT_LIST_HEAD(&net->dev_base_head);
6069
6070 net->dev_name_head = netdev_create_hash();
6071 if (net->dev_name_head == NULL)
6072 goto err_name;
6073
6074 net->dev_index_head = netdev_create_hash();
6075 if (net->dev_index_head == NULL)
6076 goto err_idx;
6077
6078 return 0;
6079
6080 err_idx:
6081 kfree(net->dev_name_head);
6082 err_name:
6083 return -ENOMEM;
6084 }
6085
6086 /**
6087 * netdev_drivername - network driver for the device
6088 * @dev: network device
6089 *
6090 * Determine network driver for device.
6091 */
netdev_drivername(const struct net_device * dev)6092 const char *netdev_drivername(const struct net_device *dev)
6093 {
6094 const struct device_driver *driver;
6095 const struct device *parent;
6096 const char *empty = "";
6097
6098 parent = dev->dev.parent;
6099 if (!parent)
6100 return empty;
6101
6102 driver = parent->driver;
6103 if (driver && driver->name)
6104 return driver->name;
6105 return empty;
6106 }
6107
__netdev_printk(const char * level,const struct net_device * dev,struct va_format * vaf)6108 static int __netdev_printk(const char *level, const struct net_device *dev,
6109 struct va_format *vaf)
6110 {
6111 int r;
6112
6113 if (dev && dev->dev.parent) {
6114 r = dev_printk_emit(level[1] - '0',
6115 dev->dev.parent,
6116 "%s %s %s: %pV",
6117 dev_driver_string(dev->dev.parent),
6118 dev_name(dev->dev.parent),
6119 netdev_name(dev), vaf);
6120 } else if (dev) {
6121 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6122 } else {
6123 r = printk("%s(NULL net_device): %pV", level, vaf);
6124 }
6125
6126 return r;
6127 }
6128
netdev_printk(const char * level,const struct net_device * dev,const char * format,...)6129 int netdev_printk(const char *level, const struct net_device *dev,
6130 const char *format, ...)
6131 {
6132 struct va_format vaf;
6133 va_list args;
6134 int r;
6135
6136 va_start(args, format);
6137
6138 vaf.fmt = format;
6139 vaf.va = &args;
6140
6141 r = __netdev_printk(level, dev, &vaf);
6142
6143 va_end(args);
6144
6145 return r;
6146 }
6147 EXPORT_SYMBOL(netdev_printk);
6148
6149 #define define_netdev_printk_level(func, level) \
6150 int func(const struct net_device *dev, const char *fmt, ...) \
6151 { \
6152 int r; \
6153 struct va_format vaf; \
6154 va_list args; \
6155 \
6156 va_start(args, fmt); \
6157 \
6158 vaf.fmt = fmt; \
6159 vaf.va = &args; \
6160 \
6161 r = __netdev_printk(level, dev, &vaf); \
6162 \
6163 va_end(args); \
6164 \
6165 return r; \
6166 } \
6167 EXPORT_SYMBOL(func);
6168
6169 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6170 define_netdev_printk_level(netdev_alert, KERN_ALERT);
6171 define_netdev_printk_level(netdev_crit, KERN_CRIT);
6172 define_netdev_printk_level(netdev_err, KERN_ERR);
6173 define_netdev_printk_level(netdev_warn, KERN_WARNING);
6174 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6175 define_netdev_printk_level(netdev_info, KERN_INFO);
6176
netdev_exit(struct net * net)6177 static void __net_exit netdev_exit(struct net *net)
6178 {
6179 kfree(net->dev_name_head);
6180 kfree(net->dev_index_head);
6181 }
6182
6183 static struct pernet_operations __net_initdata netdev_net_ops = {
6184 .init = netdev_init,
6185 .exit = netdev_exit,
6186 };
6187
default_device_exit(struct net * net)6188 static void __net_exit default_device_exit(struct net *net)
6189 {
6190 struct net_device *dev, *aux;
6191 /*
6192 * Push all migratable network devices back to the
6193 * initial network namespace
6194 */
6195 rtnl_lock();
6196 for_each_netdev_safe(net, dev, aux) {
6197 int err;
6198 char fb_name[IFNAMSIZ];
6199
6200 /* Ignore unmoveable devices (i.e. loopback) */
6201 if (dev->features & NETIF_F_NETNS_LOCAL)
6202 continue;
6203
6204 /* Leave virtual devices for the generic cleanup */
6205 if (dev->rtnl_link_ops)
6206 continue;
6207
6208 /* Push remaining network devices to init_net */
6209 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6210 err = dev_change_net_namespace(dev, &init_net, fb_name);
6211 if (err) {
6212 pr_emerg("%s: failed to move %s to init_net: %d\n",
6213 __func__, dev->name, err);
6214 BUG();
6215 }
6216 }
6217 rtnl_unlock();
6218 }
6219
default_device_exit_batch(struct list_head * net_list)6220 static void __net_exit default_device_exit_batch(struct list_head *net_list)
6221 {
6222 /* At exit all network devices most be removed from a network
6223 * namespace. Do this in the reverse order of registration.
6224 * Do this across as many network namespaces as possible to
6225 * improve batching efficiency.
6226 */
6227 struct net_device *dev;
6228 struct net *net;
6229 LIST_HEAD(dev_kill_list);
6230
6231 rtnl_lock();
6232 list_for_each_entry(net, net_list, exit_list) {
6233 for_each_netdev_reverse(net, dev) {
6234 if (dev->rtnl_link_ops)
6235 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6236 else
6237 unregister_netdevice_queue(dev, &dev_kill_list);
6238 }
6239 }
6240 unregister_netdevice_many(&dev_kill_list);
6241 list_del(&dev_kill_list);
6242 rtnl_unlock();
6243 }
6244
6245 static struct pernet_operations __net_initdata default_device_ops = {
6246 .exit = default_device_exit,
6247 .exit_batch = default_device_exit_batch,
6248 };
6249
6250 /*
6251 * Initialize the DEV module. At boot time this walks the device list and
6252 * unhooks any devices that fail to initialise (normally hardware not
6253 * present) and leaves us with a valid list of present and active devices.
6254 *
6255 */
6256
6257 /*
6258 * This is called single threaded during boot, so no need
6259 * to take the rtnl semaphore.
6260 */
net_dev_init(void)6261 static int __init net_dev_init(void)
6262 {
6263 int i, rc = -ENOMEM;
6264
6265 BUG_ON(!dev_boot_phase);
6266
6267 if (dev_proc_init())
6268 goto out;
6269
6270 if (netdev_kobject_init())
6271 goto out;
6272
6273 INIT_LIST_HEAD(&ptype_all);
6274 for (i = 0; i < PTYPE_HASH_SIZE; i++)
6275 INIT_LIST_HEAD(&ptype_base[i]);
6276
6277 INIT_LIST_HEAD(&offload_base);
6278
6279 if (register_pernet_subsys(&netdev_net_ops))
6280 goto out;
6281
6282 /*
6283 * Initialise the packet receive queues.
6284 */
6285
6286 for_each_possible_cpu(i) {
6287 struct softnet_data *sd = &per_cpu(softnet_data, i);
6288
6289 memset(sd, 0, sizeof(*sd));
6290 skb_queue_head_init(&sd->input_pkt_queue);
6291 skb_queue_head_init(&sd->process_queue);
6292 sd->completion_queue = NULL;
6293 INIT_LIST_HEAD(&sd->poll_list);
6294 sd->output_queue = NULL;
6295 sd->output_queue_tailp = &sd->output_queue;
6296 #ifdef CONFIG_RPS
6297 sd->csd.func = rps_trigger_softirq;
6298 sd->csd.info = sd;
6299 sd->csd.flags = 0;
6300 sd->cpu = i;
6301 #endif
6302
6303 sd->backlog.poll = process_backlog;
6304 sd->backlog.weight = weight_p;
6305 sd->backlog.gro_list = NULL;
6306 sd->backlog.gro_count = 0;
6307 }
6308
6309 dev_boot_phase = 0;
6310
6311 /* The loopback device is special if any other network devices
6312 * is present in a network namespace the loopback device must
6313 * be present. Since we now dynamically allocate and free the
6314 * loopback device ensure this invariant is maintained by
6315 * keeping the loopback device as the first device on the
6316 * list of network devices. Ensuring the loopback devices
6317 * is the first device that appears and the last network device
6318 * that disappears.
6319 */
6320 if (register_pernet_device(&loopback_net_ops))
6321 goto out;
6322
6323 if (register_pernet_device(&default_device_ops))
6324 goto out;
6325
6326 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6327 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
6328
6329 hotcpu_notifier(dev_cpu_callback, 0);
6330 dst_init();
6331 rc = 0;
6332 out:
6333 return rc;
6334 }
6335
6336 subsys_initcall(net_dev_init);
6337