• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * 	NET3	Protocol independent device support routines.
3  *
4  *		This program is free software; you can redistribute it and/or
5  *		modify it under the terms of the GNU General Public License
6  *		as published by the Free Software Foundation; either version
7  *		2 of the License, or (at your option) any later version.
8  *
9  *	Derived from the non IP parts of dev.c 1.0.19
10  * 		Authors:	Ross Biro
11  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *
14  *	Additional Authors:
15  *		Florian la Roche <rzsfl@rz.uni-sb.de>
16  *		Alan Cox <gw4pts@gw4pts.ampr.org>
17  *		David Hinds <dahinds@users.sourceforge.net>
18  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19  *		Adam Sulmicki <adam@cfar.umd.edu>
20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
21  *
22  *	Changes:
23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
24  *              			to 2 if register_netdev gets called
25  *              			before net_dev_init & also removed a
26  *              			few lines of code in the process.
27  *		Alan Cox	:	device private ioctl copies fields back.
28  *		Alan Cox	:	Transmit queue code does relevant
29  *					stunts to keep the queue safe.
30  *		Alan Cox	:	Fixed double lock.
31  *		Alan Cox	:	Fixed promisc NULL pointer trap
32  *		????????	:	Support the full private ioctl range
33  *		Alan Cox	:	Moved ioctl permission check into
34  *					drivers
35  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
36  *		Alan Cox	:	100 backlog just doesn't cut it when
37  *					you start doing multicast video 8)
38  *		Alan Cox	:	Rewrote net_bh and list manager.
39  *		Alan Cox	: 	Fix ETH_P_ALL echoback lengths.
40  *		Alan Cox	:	Took out transmit every packet pass
41  *					Saved a few bytes in the ioctl handler
42  *		Alan Cox	:	Network driver sets packet type before
43  *					calling netif_rx. Saves a function
44  *					call a packet.
45  *		Alan Cox	:	Hashed net_bh()
46  *		Richard Kooijman:	Timestamp fixes.
47  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
48  *		Alan Cox	:	Device lock protection.
49  *		Alan Cox	: 	Fixed nasty side effect of device close
50  *					changes.
51  *		Rudi Cilibrasi	:	Pass the right thing to
52  *					set_mac_address()
53  *		Dave Miller	:	32bit quantity for the device lock to
54  *					make it work out on a Sparc.
55  *		Bjorn Ekwall	:	Added KERNELD hack.
56  *		Alan Cox	:	Cleaned up the backlog initialise.
57  *		Craig Metz	:	SIOCGIFCONF fix if space for under
58  *					1 device.
59  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
60  *					is no device open function.
61  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
62  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
63  *		Cyrus Durgin	:	Cleaned for KMOD
64  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
65  *					A network device unload needs to purge
66  *					the backlog queue.
67  *	Paul Rusty Russell	:	SIOCSIFNAME
68  *              Pekka Riikonen  :	Netdev boot-time settings code
69  *              Andrew Morton   :       Make unregister_netdevice wait
70  *              			indefinitely on dev->refcnt
71  * 		J Hadi Salim	:	- Backlog queue sampling
72  *				        - netif_rx() feedback
73  */
74 
75 #include <asm/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <net/net_namespace.h>
98 #include <net/sock.h>
99 #include <linux/rtnetlink.h>
100 #include <linux/stat.h>
101 #include <net/dst.h>
102 #include <net/pkt_sched.h>
103 #include <net/checksum.h>
104 #include <net/xfrm.h>
105 #include <linux/highmem.h>
106 #include <linux/init.h>
107 #include <linux/module.h>
108 #include <linux/netpoll.h>
109 #include <linux/rcupdate.h>
110 #include <linux/delay.h>
111 #include <net/iw_handler.h>
112 #include <asm/current.h>
113 #include <linux/audit.h>
114 #include <linux/dmaengine.h>
115 #include <linux/err.h>
116 #include <linux/ctype.h>
117 #include <linux/if_arp.h>
118 #include <linux/if_vlan.h>
119 #include <linux/ip.h>
120 #include <net/ip.h>
121 #include <linux/ipv6.h>
122 #include <linux/in.h>
123 #include <linux/jhash.h>
124 #include <linux/random.h>
125 #include <trace/events/napi.h>
126 #include <trace/events/net.h>
127 #include <trace/events/skb.h>
128 #include <linux/pci.h>
129 #include <linux/inetdevice.h>
130 #include <linux/cpu_rmap.h>
131 #include <linux/static_key.h>
132 
133 #include "net-sysfs.h"
134 
135 /* Instead of increasing this, you should create a hash table. */
136 #define MAX_GRO_SKBS 8
137 
138 /* This should be increased if a protocol with a bigger head is added. */
139 #define GRO_MAX_HEAD (MAX_HEADER + 128)
140 
141 static DEFINE_SPINLOCK(ptype_lock);
142 static DEFINE_SPINLOCK(offload_lock);
143 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
144 struct list_head ptype_all __read_mostly;	/* Taps */
145 static struct list_head offload_base __read_mostly;
146 
147 /*
148  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
149  * semaphore.
150  *
151  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
152  *
153  * Writers must hold the rtnl semaphore while they loop through the
154  * dev_base_head list, and hold dev_base_lock for writing when they do the
155  * actual updates.  This allows pure readers to access the list even
156  * while a writer is preparing to update it.
157  *
158  * To put it another way, dev_base_lock is held for writing only to
159  * protect against pure readers; the rtnl semaphore provides the
160  * protection against other writers.
161  *
162  * See, for example usages, register_netdevice() and
163  * unregister_netdevice(), which must be called with the rtnl
164  * semaphore held.
165  */
166 DEFINE_RWLOCK(dev_base_lock);
167 EXPORT_SYMBOL(dev_base_lock);
168 
169 seqcount_t devnet_rename_seq;
170 
dev_base_seq_inc(struct net * net)171 static inline void dev_base_seq_inc(struct net *net)
172 {
173 	while (++net->dev_base_seq == 0);
174 }
175 
dev_name_hash(struct net * net,const char * name)176 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
177 {
178 	unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
179 
180 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
181 }
182 
dev_index_hash(struct net * net,int ifindex)183 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
184 {
185 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
186 }
187 
rps_lock(struct softnet_data * sd)188 static inline void rps_lock(struct softnet_data *sd)
189 {
190 #ifdef CONFIG_RPS
191 	spin_lock(&sd->input_pkt_queue.lock);
192 #endif
193 }
194 
rps_unlock(struct softnet_data * sd)195 static inline void rps_unlock(struct softnet_data *sd)
196 {
197 #ifdef CONFIG_RPS
198 	spin_unlock(&sd->input_pkt_queue.lock);
199 #endif
200 }
201 
202 /* Device list insertion */
list_netdevice(struct net_device * dev)203 static void list_netdevice(struct net_device *dev)
204 {
205 	struct net *net = dev_net(dev);
206 
207 	ASSERT_RTNL();
208 
209 	write_lock_bh(&dev_base_lock);
210 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
211 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
212 	hlist_add_head_rcu(&dev->index_hlist,
213 			   dev_index_hash(net, dev->ifindex));
214 	write_unlock_bh(&dev_base_lock);
215 
216 	dev_base_seq_inc(net);
217 }
218 
219 /* Device list removal
220  * caller must respect a RCU grace period before freeing/reusing dev
221  */
unlist_netdevice(struct net_device * dev)222 static void unlist_netdevice(struct net_device *dev)
223 {
224 	ASSERT_RTNL();
225 
226 	/* Unlink dev from the device chain */
227 	write_lock_bh(&dev_base_lock);
228 	list_del_rcu(&dev->dev_list);
229 	hlist_del_rcu(&dev->name_hlist);
230 	hlist_del_rcu(&dev->index_hlist);
231 	write_unlock_bh(&dev_base_lock);
232 
233 	dev_base_seq_inc(dev_net(dev));
234 }
235 
236 /*
237  *	Our notifier list
238  */
239 
240 static RAW_NOTIFIER_HEAD(netdev_chain);
241 
242 /*
243  *	Device drivers call our routines to queue packets here. We empty the
244  *	queue in the local softnet handler.
245  */
246 
247 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
248 EXPORT_PER_CPU_SYMBOL(softnet_data);
249 
250 #ifdef CONFIG_LOCKDEP
251 /*
252  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
253  * according to dev->type
254  */
255 static const unsigned short netdev_lock_type[] =
256 	{ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
257 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
258 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
259 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
260 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
261 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
262 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
263 	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
264 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
265 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
266 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
267 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
268 	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
269 	 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
270 	 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
271 
272 static const char *const netdev_lock_name[] =
273 	{"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
274 	 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
275 	 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
276 	 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
277 	 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
278 	 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
279 	 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
280 	 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
281 	 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
282 	 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
283 	 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
284 	 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
285 	 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
286 	 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
287 	 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
288 
289 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
290 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
291 
netdev_lock_pos(unsigned short dev_type)292 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
293 {
294 	int i;
295 
296 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
297 		if (netdev_lock_type[i] == dev_type)
298 			return i;
299 	/* the last key is used by default */
300 	return ARRAY_SIZE(netdev_lock_type) - 1;
301 }
302 
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)303 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
304 						 unsigned short dev_type)
305 {
306 	int i;
307 
308 	i = netdev_lock_pos(dev_type);
309 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
310 				   netdev_lock_name[i]);
311 }
312 
netdev_set_addr_lockdep_class(struct net_device * dev)313 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
314 {
315 	int i;
316 
317 	i = netdev_lock_pos(dev->type);
318 	lockdep_set_class_and_name(&dev->addr_list_lock,
319 				   &netdev_addr_lock_key[i],
320 				   netdev_lock_name[i]);
321 }
322 #else
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)323 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
324 						 unsigned short dev_type)
325 {
326 }
netdev_set_addr_lockdep_class(struct net_device * dev)327 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
328 {
329 }
330 #endif
331 
332 /*******************************************************************************
333 
334 		Protocol management and registration routines
335 
336 *******************************************************************************/
337 
338 /*
339  *	Add a protocol ID to the list. Now that the input handler is
340  *	smarter we can dispense with all the messy stuff that used to be
341  *	here.
342  *
343  *	BEWARE!!! Protocol handlers, mangling input packets,
344  *	MUST BE last in hash buckets and checking protocol handlers
345  *	MUST start from promiscuous ptype_all chain in net_bh.
346  *	It is true now, do not change it.
347  *	Explanation follows: if protocol handler, mangling packet, will
348  *	be the first on list, it is not able to sense, that packet
349  *	is cloned and should be copied-on-write, so that it will
350  *	change it and subsequent readers will get broken packet.
351  *							--ANK (980803)
352  */
353 
ptype_head(const struct packet_type * pt)354 static inline struct list_head *ptype_head(const struct packet_type *pt)
355 {
356 	if (pt->type == htons(ETH_P_ALL))
357 		return &ptype_all;
358 	else
359 		return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
360 }
361 
362 /**
363  *	dev_add_pack - add packet handler
364  *	@pt: packet type declaration
365  *
366  *	Add a protocol handler to the networking stack. The passed &packet_type
367  *	is linked into kernel lists and may not be freed until it has been
368  *	removed from the kernel lists.
369  *
370  *	This call does not sleep therefore it can not
371  *	guarantee all CPU's that are in middle of receiving packets
372  *	will see the new packet type (until the next received packet).
373  */
374 
dev_add_pack(struct packet_type * pt)375 void dev_add_pack(struct packet_type *pt)
376 {
377 	struct list_head *head = ptype_head(pt);
378 
379 	spin_lock(&ptype_lock);
380 	list_add_rcu(&pt->list, head);
381 	spin_unlock(&ptype_lock);
382 }
383 EXPORT_SYMBOL(dev_add_pack);
384 
385 /**
386  *	__dev_remove_pack	 - remove packet handler
387  *	@pt: packet type declaration
388  *
389  *	Remove a protocol handler that was previously added to the kernel
390  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
391  *	from the kernel lists and can be freed or reused once this function
392  *	returns.
393  *
394  *      The packet type might still be in use by receivers
395  *	and must not be freed until after all the CPU's have gone
396  *	through a quiescent state.
397  */
__dev_remove_pack(struct packet_type * pt)398 void __dev_remove_pack(struct packet_type *pt)
399 {
400 	struct list_head *head = ptype_head(pt);
401 	struct packet_type *pt1;
402 
403 	spin_lock(&ptype_lock);
404 
405 	list_for_each_entry(pt1, head, list) {
406 		if (pt == pt1) {
407 			list_del_rcu(&pt->list);
408 			goto out;
409 		}
410 	}
411 
412 	pr_warn("dev_remove_pack: %p not found\n", pt);
413 out:
414 	spin_unlock(&ptype_lock);
415 }
416 EXPORT_SYMBOL(__dev_remove_pack);
417 
418 /**
419  *	dev_remove_pack	 - remove packet handler
420  *	@pt: packet type declaration
421  *
422  *	Remove a protocol handler that was previously added to the kernel
423  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
424  *	from the kernel lists and can be freed or reused once this function
425  *	returns.
426  *
427  *	This call sleeps to guarantee that no CPU is looking at the packet
428  *	type after return.
429  */
dev_remove_pack(struct packet_type * pt)430 void dev_remove_pack(struct packet_type *pt)
431 {
432 	__dev_remove_pack(pt);
433 
434 	synchronize_net();
435 }
436 EXPORT_SYMBOL(dev_remove_pack);
437 
438 
439 /**
440  *	dev_add_offload - register offload handlers
441  *	@po: protocol offload declaration
442  *
443  *	Add protocol offload handlers to the networking stack. The passed
444  *	&proto_offload is linked into kernel lists and may not be freed until
445  *	it has been removed from the kernel lists.
446  *
447  *	This call does not sleep therefore it can not
448  *	guarantee all CPU's that are in middle of receiving packets
449  *	will see the new offload handlers (until the next received packet).
450  */
dev_add_offload(struct packet_offload * po)451 void dev_add_offload(struct packet_offload *po)
452 {
453 	struct list_head *head = &offload_base;
454 
455 	spin_lock(&offload_lock);
456 	list_add_rcu(&po->list, head);
457 	spin_unlock(&offload_lock);
458 }
459 EXPORT_SYMBOL(dev_add_offload);
460 
461 /**
462  *	__dev_remove_offload	 - remove offload handler
463  *	@po: packet offload declaration
464  *
465  *	Remove a protocol offload handler that was previously added to the
466  *	kernel offload handlers by dev_add_offload(). The passed &offload_type
467  *	is removed from the kernel lists and can be freed or reused once this
468  *	function returns.
469  *
470  *      The packet type might still be in use by receivers
471  *	and must not be freed until after all the CPU's have gone
472  *	through a quiescent state.
473  */
__dev_remove_offload(struct packet_offload * po)474 void __dev_remove_offload(struct packet_offload *po)
475 {
476 	struct list_head *head = &offload_base;
477 	struct packet_offload *po1;
478 
479 	spin_lock(&offload_lock);
480 
481 	list_for_each_entry(po1, head, list) {
482 		if (po == po1) {
483 			list_del_rcu(&po->list);
484 			goto out;
485 		}
486 	}
487 
488 	pr_warn("dev_remove_offload: %p not found\n", po);
489 out:
490 	spin_unlock(&offload_lock);
491 }
492 EXPORT_SYMBOL(__dev_remove_offload);
493 
494 /**
495  *	dev_remove_offload	 - remove packet offload handler
496  *	@po: packet offload declaration
497  *
498  *	Remove a packet offload handler that was previously added to the kernel
499  *	offload handlers by dev_add_offload(). The passed &offload_type is
500  *	removed from the kernel lists and can be freed or reused once this
501  *	function returns.
502  *
503  *	This call sleeps to guarantee that no CPU is looking at the packet
504  *	type after return.
505  */
dev_remove_offload(struct packet_offload * po)506 void dev_remove_offload(struct packet_offload *po)
507 {
508 	__dev_remove_offload(po);
509 
510 	synchronize_net();
511 }
512 EXPORT_SYMBOL(dev_remove_offload);
513 
514 /******************************************************************************
515 
516 		      Device Boot-time Settings Routines
517 
518 *******************************************************************************/
519 
520 /* Boot time configuration table */
521 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
522 
523 /**
524  *	netdev_boot_setup_add	- add new setup entry
525  *	@name: name of the device
526  *	@map: configured settings for the device
527  *
528  *	Adds new setup entry to the dev_boot_setup list.  The function
529  *	returns 0 on error and 1 on success.  This is a generic routine to
530  *	all netdevices.
531  */
netdev_boot_setup_add(char * name,struct ifmap * map)532 static int netdev_boot_setup_add(char *name, struct ifmap *map)
533 {
534 	struct netdev_boot_setup *s;
535 	int i;
536 
537 	s = dev_boot_setup;
538 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
539 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
540 			memset(s[i].name, 0, sizeof(s[i].name));
541 			strlcpy(s[i].name, name, IFNAMSIZ);
542 			memcpy(&s[i].map, map, sizeof(s[i].map));
543 			break;
544 		}
545 	}
546 
547 	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
548 }
549 
550 /**
551  *	netdev_boot_setup_check	- check boot time settings
552  *	@dev: the netdevice
553  *
554  * 	Check boot time settings for the device.
555  *	The found settings are set for the device to be used
556  *	later in the device probing.
557  *	Returns 0 if no settings found, 1 if they are.
558  */
netdev_boot_setup_check(struct net_device * dev)559 int netdev_boot_setup_check(struct net_device *dev)
560 {
561 	struct netdev_boot_setup *s = dev_boot_setup;
562 	int i;
563 
564 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
565 		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
566 		    !strcmp(dev->name, s[i].name)) {
567 			dev->irq 	= s[i].map.irq;
568 			dev->base_addr 	= s[i].map.base_addr;
569 			dev->mem_start 	= s[i].map.mem_start;
570 			dev->mem_end 	= s[i].map.mem_end;
571 			return 1;
572 		}
573 	}
574 	return 0;
575 }
576 EXPORT_SYMBOL(netdev_boot_setup_check);
577 
578 
579 /**
580  *	netdev_boot_base	- get address from boot time settings
581  *	@prefix: prefix for network device
582  *	@unit: id for network device
583  *
584  * 	Check boot time settings for the base address of device.
585  *	The found settings are set for the device to be used
586  *	later in the device probing.
587  *	Returns 0 if no settings found.
588  */
netdev_boot_base(const char * prefix,int unit)589 unsigned long netdev_boot_base(const char *prefix, int unit)
590 {
591 	const struct netdev_boot_setup *s = dev_boot_setup;
592 	char name[IFNAMSIZ];
593 	int i;
594 
595 	sprintf(name, "%s%d", prefix, unit);
596 
597 	/*
598 	 * If device already registered then return base of 1
599 	 * to indicate not to probe for this interface
600 	 */
601 	if (__dev_get_by_name(&init_net, name))
602 		return 1;
603 
604 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
605 		if (!strcmp(name, s[i].name))
606 			return s[i].map.base_addr;
607 	return 0;
608 }
609 
610 /*
611  * Saves at boot time configured settings for any netdevice.
612  */
netdev_boot_setup(char * str)613 int __init netdev_boot_setup(char *str)
614 {
615 	int ints[5];
616 	struct ifmap map;
617 
618 	str = get_options(str, ARRAY_SIZE(ints), ints);
619 	if (!str || !*str)
620 		return 0;
621 
622 	/* Save settings */
623 	memset(&map, 0, sizeof(map));
624 	if (ints[0] > 0)
625 		map.irq = ints[1];
626 	if (ints[0] > 1)
627 		map.base_addr = ints[2];
628 	if (ints[0] > 2)
629 		map.mem_start = ints[3];
630 	if (ints[0] > 3)
631 		map.mem_end = ints[4];
632 
633 	/* Add new entry to the list */
634 	return netdev_boot_setup_add(str, &map);
635 }
636 
637 __setup("netdev=", netdev_boot_setup);
638 
639 /*******************************************************************************
640 
641 			    Device Interface Subroutines
642 
643 *******************************************************************************/
644 
645 /**
646  *	__dev_get_by_name	- find a device by its name
647  *	@net: the applicable net namespace
648  *	@name: name to find
649  *
650  *	Find an interface by name. Must be called under RTNL semaphore
651  *	or @dev_base_lock. If the name is found a pointer to the device
652  *	is returned. If the name is not found then %NULL is returned. The
653  *	reference counters are not incremented so the caller must be
654  *	careful with locks.
655  */
656 
__dev_get_by_name(struct net * net,const char * name)657 struct net_device *__dev_get_by_name(struct net *net, const char *name)
658 {
659 	struct net_device *dev;
660 	struct hlist_head *head = dev_name_hash(net, name);
661 
662 	hlist_for_each_entry(dev, head, name_hlist)
663 		if (!strncmp(dev->name, name, IFNAMSIZ))
664 			return dev;
665 
666 	return NULL;
667 }
668 EXPORT_SYMBOL(__dev_get_by_name);
669 
670 /**
671  *	dev_get_by_name_rcu	- find a device by its name
672  *	@net: the applicable net namespace
673  *	@name: name to find
674  *
675  *	Find an interface by name.
676  *	If the name is found a pointer to the device is returned.
677  * 	If the name is not found then %NULL is returned.
678  *	The reference counters are not incremented so the caller must be
679  *	careful with locks. The caller must hold RCU lock.
680  */
681 
dev_get_by_name_rcu(struct net * net,const char * name)682 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
683 {
684 	struct net_device *dev;
685 	struct hlist_head *head = dev_name_hash(net, name);
686 
687 	hlist_for_each_entry_rcu(dev, head, name_hlist)
688 		if (!strncmp(dev->name, name, IFNAMSIZ))
689 			return dev;
690 
691 	return NULL;
692 }
693 EXPORT_SYMBOL(dev_get_by_name_rcu);
694 
695 /**
696  *	dev_get_by_name		- find a device by its name
697  *	@net: the applicable net namespace
698  *	@name: name to find
699  *
700  *	Find an interface by name. This can be called from any
701  *	context and does its own locking. The returned handle has
702  *	the usage count incremented and the caller must use dev_put() to
703  *	release it when it is no longer needed. %NULL is returned if no
704  *	matching device is found.
705  */
706 
dev_get_by_name(struct net * net,const char * name)707 struct net_device *dev_get_by_name(struct net *net, const char *name)
708 {
709 	struct net_device *dev;
710 
711 	rcu_read_lock();
712 	dev = dev_get_by_name_rcu(net, name);
713 	if (dev)
714 		dev_hold(dev);
715 	rcu_read_unlock();
716 	return dev;
717 }
718 EXPORT_SYMBOL(dev_get_by_name);
719 
720 /**
721  *	__dev_get_by_index - find a device by its ifindex
722  *	@net: the applicable net namespace
723  *	@ifindex: index of device
724  *
725  *	Search for an interface by index. Returns %NULL if the device
726  *	is not found or a pointer to the device. The device has not
727  *	had its reference counter increased so the caller must be careful
728  *	about locking. The caller must hold either the RTNL semaphore
729  *	or @dev_base_lock.
730  */
731 
__dev_get_by_index(struct net * net,int ifindex)732 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
733 {
734 	struct net_device *dev;
735 	struct hlist_head *head = dev_index_hash(net, ifindex);
736 
737 	hlist_for_each_entry(dev, head, index_hlist)
738 		if (dev->ifindex == ifindex)
739 			return dev;
740 
741 	return NULL;
742 }
743 EXPORT_SYMBOL(__dev_get_by_index);
744 
745 /**
746  *	dev_get_by_index_rcu - find a device by its ifindex
747  *	@net: the applicable net namespace
748  *	@ifindex: index of device
749  *
750  *	Search for an interface by index. Returns %NULL if the device
751  *	is not found or a pointer to the device. The device has not
752  *	had its reference counter increased so the caller must be careful
753  *	about locking. The caller must hold RCU lock.
754  */
755 
dev_get_by_index_rcu(struct net * net,int ifindex)756 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
757 {
758 	struct net_device *dev;
759 	struct hlist_head *head = dev_index_hash(net, ifindex);
760 
761 	hlist_for_each_entry_rcu(dev, head, index_hlist)
762 		if (dev->ifindex == ifindex)
763 			return dev;
764 
765 	return NULL;
766 }
767 EXPORT_SYMBOL(dev_get_by_index_rcu);
768 
769 
770 /**
771  *	dev_get_by_index - find a device by its ifindex
772  *	@net: the applicable net namespace
773  *	@ifindex: index of device
774  *
775  *	Search for an interface by index. Returns NULL if the device
776  *	is not found or a pointer to the device. The device returned has
777  *	had a reference added and the pointer is safe until the user calls
778  *	dev_put to indicate they have finished with it.
779  */
780 
dev_get_by_index(struct net * net,int ifindex)781 struct net_device *dev_get_by_index(struct net *net, int ifindex)
782 {
783 	struct net_device *dev;
784 
785 	rcu_read_lock();
786 	dev = dev_get_by_index_rcu(net, ifindex);
787 	if (dev)
788 		dev_hold(dev);
789 	rcu_read_unlock();
790 	return dev;
791 }
792 EXPORT_SYMBOL(dev_get_by_index);
793 
794 /**
795  *	netdev_get_name - get a netdevice name, knowing its ifindex.
796  *	@net: network namespace
797  *	@name: a pointer to the buffer where the name will be stored.
798  *	@ifindex: the ifindex of the interface to get the name from.
799  *
800  *	The use of raw_seqcount_begin() and cond_resched() before
801  *	retrying is required as we want to give the writers a chance
802  *	to complete when CONFIG_PREEMPT is not set.
803  */
netdev_get_name(struct net * net,char * name,int ifindex)804 int netdev_get_name(struct net *net, char *name, int ifindex)
805 {
806 	struct net_device *dev;
807 	unsigned int seq;
808 
809 retry:
810 	seq = raw_seqcount_begin(&devnet_rename_seq);
811 	rcu_read_lock();
812 	dev = dev_get_by_index_rcu(net, ifindex);
813 	if (!dev) {
814 		rcu_read_unlock();
815 		return -ENODEV;
816 	}
817 
818 	strcpy(name, dev->name);
819 	rcu_read_unlock();
820 	if (read_seqcount_retry(&devnet_rename_seq, seq)) {
821 		cond_resched();
822 		goto retry;
823 	}
824 
825 	return 0;
826 }
827 
828 /**
829  *	dev_getbyhwaddr_rcu - find a device by its hardware address
830  *	@net: the applicable net namespace
831  *	@type: media type of device
832  *	@ha: hardware address
833  *
834  *	Search for an interface by MAC address. Returns NULL if the device
835  *	is not found or a pointer to the device.
836  *	The caller must hold RCU or RTNL.
837  *	The returned device has not had its ref count increased
838  *	and the caller must therefore be careful about locking
839  *
840  */
841 
dev_getbyhwaddr_rcu(struct net * net,unsigned short type,const char * ha)842 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
843 				       const char *ha)
844 {
845 	struct net_device *dev;
846 
847 	for_each_netdev_rcu(net, dev)
848 		if (dev->type == type &&
849 		    !memcmp(dev->dev_addr, ha, dev->addr_len))
850 			return dev;
851 
852 	return NULL;
853 }
854 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
855 
__dev_getfirstbyhwtype(struct net * net,unsigned short type)856 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
857 {
858 	struct net_device *dev;
859 
860 	ASSERT_RTNL();
861 	for_each_netdev(net, dev)
862 		if (dev->type == type)
863 			return dev;
864 
865 	return NULL;
866 }
867 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
868 
dev_getfirstbyhwtype(struct net * net,unsigned short type)869 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
870 {
871 	struct net_device *dev, *ret = NULL;
872 
873 	rcu_read_lock();
874 	for_each_netdev_rcu(net, dev)
875 		if (dev->type == type) {
876 			dev_hold(dev);
877 			ret = dev;
878 			break;
879 		}
880 	rcu_read_unlock();
881 	return ret;
882 }
883 EXPORT_SYMBOL(dev_getfirstbyhwtype);
884 
885 /**
886  *	dev_get_by_flags_rcu - find any device with given flags
887  *	@net: the applicable net namespace
888  *	@if_flags: IFF_* values
889  *	@mask: bitmask of bits in if_flags to check
890  *
891  *	Search for any interface with the given flags. Returns NULL if a device
892  *	is not found or a pointer to the device. Must be called inside
893  *	rcu_read_lock(), and result refcount is unchanged.
894  */
895 
dev_get_by_flags_rcu(struct net * net,unsigned short if_flags,unsigned short mask)896 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
897 				    unsigned short mask)
898 {
899 	struct net_device *dev, *ret;
900 
901 	ret = NULL;
902 	for_each_netdev_rcu(net, dev) {
903 		if (((dev->flags ^ if_flags) & mask) == 0) {
904 			ret = dev;
905 			break;
906 		}
907 	}
908 	return ret;
909 }
910 EXPORT_SYMBOL(dev_get_by_flags_rcu);
911 
912 /**
913  *	dev_valid_name - check if name is okay for network device
914  *	@name: name string
915  *
916  *	Network device names need to be valid file names to
917  *	to allow sysfs to work.  We also disallow any kind of
918  *	whitespace.
919  */
dev_valid_name(const char * name)920 bool dev_valid_name(const char *name)
921 {
922 	if (*name == '\0')
923 		return false;
924 	if (strlen(name) >= IFNAMSIZ)
925 		return false;
926 	if (!strcmp(name, ".") || !strcmp(name, ".."))
927 		return false;
928 
929 	while (*name) {
930 		if (*name == '/' || isspace(*name))
931 			return false;
932 		name++;
933 	}
934 	return true;
935 }
936 EXPORT_SYMBOL(dev_valid_name);
937 
938 /**
939  *	__dev_alloc_name - allocate a name for a device
940  *	@net: network namespace to allocate the device name in
941  *	@name: name format string
942  *	@buf:  scratch buffer and result name string
943  *
944  *	Passed a format string - eg "lt%d" it will try and find a suitable
945  *	id. It scans list of devices to build up a free map, then chooses
946  *	the first empty slot. The caller must hold the dev_base or rtnl lock
947  *	while allocating the name and adding the device in order to avoid
948  *	duplicates.
949  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
950  *	Returns the number of the unit assigned or a negative errno code.
951  */
952 
__dev_alloc_name(struct net * net,const char * name,char * buf)953 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
954 {
955 	int i = 0;
956 	const char *p;
957 	const int max_netdevices = 8*PAGE_SIZE;
958 	unsigned long *inuse;
959 	struct net_device *d;
960 
961 	p = strnchr(name, IFNAMSIZ-1, '%');
962 	if (p) {
963 		/*
964 		 * Verify the string as this thing may have come from
965 		 * the user.  There must be either one "%d" and no other "%"
966 		 * characters.
967 		 */
968 		if (p[1] != 'd' || strchr(p + 2, '%'))
969 			return -EINVAL;
970 
971 		/* Use one page as a bit array of possible slots */
972 		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
973 		if (!inuse)
974 			return -ENOMEM;
975 
976 		for_each_netdev(net, d) {
977 			if (!sscanf(d->name, name, &i))
978 				continue;
979 			if (i < 0 || i >= max_netdevices)
980 				continue;
981 
982 			/*  avoid cases where sscanf is not exact inverse of printf */
983 			snprintf(buf, IFNAMSIZ, name, i);
984 			if (!strncmp(buf, d->name, IFNAMSIZ))
985 				set_bit(i, inuse);
986 		}
987 
988 		i = find_first_zero_bit(inuse, max_netdevices);
989 		free_page((unsigned long) inuse);
990 	}
991 
992 	if (buf != name)
993 		snprintf(buf, IFNAMSIZ, name, i);
994 	if (!__dev_get_by_name(net, buf))
995 		return i;
996 
997 	/* It is possible to run out of possible slots
998 	 * when the name is long and there isn't enough space left
999 	 * for the digits, or if all bits are used.
1000 	 */
1001 	return -ENFILE;
1002 }
1003 
1004 /**
1005  *	dev_alloc_name - allocate a name for a device
1006  *	@dev: device
1007  *	@name: name format string
1008  *
1009  *	Passed a format string - eg "lt%d" it will try and find a suitable
1010  *	id. It scans list of devices to build up a free map, then chooses
1011  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1012  *	while allocating the name and adding the device in order to avoid
1013  *	duplicates.
1014  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1015  *	Returns the number of the unit assigned or a negative errno code.
1016  */
1017 
dev_alloc_name(struct net_device * dev,const char * name)1018 int dev_alloc_name(struct net_device *dev, const char *name)
1019 {
1020 	char buf[IFNAMSIZ];
1021 	struct net *net;
1022 	int ret;
1023 
1024 	BUG_ON(!dev_net(dev));
1025 	net = dev_net(dev);
1026 	ret = __dev_alloc_name(net, name, buf);
1027 	if (ret >= 0)
1028 		strlcpy(dev->name, buf, IFNAMSIZ);
1029 	return ret;
1030 }
1031 EXPORT_SYMBOL(dev_alloc_name);
1032 
dev_alloc_name_ns(struct net * net,struct net_device * dev,const char * name)1033 static int dev_alloc_name_ns(struct net *net,
1034 			     struct net_device *dev,
1035 			     const char *name)
1036 {
1037 	char buf[IFNAMSIZ];
1038 	int ret;
1039 
1040 	ret = __dev_alloc_name(net, name, buf);
1041 	if (ret >= 0)
1042 		strlcpy(dev->name, buf, IFNAMSIZ);
1043 	return ret;
1044 }
1045 
dev_get_valid_name(struct net * net,struct net_device * dev,const char * name)1046 static int dev_get_valid_name(struct net *net,
1047 			      struct net_device *dev,
1048 			      const char *name)
1049 {
1050 	BUG_ON(!net);
1051 
1052 	if (!dev_valid_name(name))
1053 		return -EINVAL;
1054 
1055 	if (strchr(name, '%'))
1056 		return dev_alloc_name_ns(net, dev, name);
1057 	else if (__dev_get_by_name(net, name))
1058 		return -EEXIST;
1059 	else if (dev->name != name)
1060 		strlcpy(dev->name, name, IFNAMSIZ);
1061 
1062 	return 0;
1063 }
1064 
1065 /**
1066  *	dev_change_name - change name of a device
1067  *	@dev: device
1068  *	@newname: name (or format string) must be at least IFNAMSIZ
1069  *
1070  *	Change name of a device, can pass format strings "eth%d".
1071  *	for wildcarding.
1072  */
dev_change_name(struct net_device * dev,const char * newname)1073 int dev_change_name(struct net_device *dev, const char *newname)
1074 {
1075 	char oldname[IFNAMSIZ];
1076 	int err = 0;
1077 	int ret;
1078 	struct net *net;
1079 
1080 	ASSERT_RTNL();
1081 	BUG_ON(!dev_net(dev));
1082 
1083 	net = dev_net(dev);
1084 	if (dev->flags & IFF_UP)
1085 		return -EBUSY;
1086 
1087 	write_seqcount_begin(&devnet_rename_seq);
1088 
1089 	if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1090 		write_seqcount_end(&devnet_rename_seq);
1091 		return 0;
1092 	}
1093 
1094 	memcpy(oldname, dev->name, IFNAMSIZ);
1095 
1096 	err = dev_get_valid_name(net, dev, newname);
1097 	if (err < 0) {
1098 		write_seqcount_end(&devnet_rename_seq);
1099 		return err;
1100 	}
1101 
1102 rollback:
1103 	ret = device_rename(&dev->dev, dev->name);
1104 	if (ret) {
1105 		memcpy(dev->name, oldname, IFNAMSIZ);
1106 		write_seqcount_end(&devnet_rename_seq);
1107 		return ret;
1108 	}
1109 
1110 	write_seqcount_end(&devnet_rename_seq);
1111 
1112 	write_lock_bh(&dev_base_lock);
1113 	hlist_del_rcu(&dev->name_hlist);
1114 	write_unlock_bh(&dev_base_lock);
1115 
1116 	synchronize_rcu();
1117 
1118 	write_lock_bh(&dev_base_lock);
1119 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1120 	write_unlock_bh(&dev_base_lock);
1121 
1122 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1123 	ret = notifier_to_errno(ret);
1124 
1125 	if (ret) {
1126 		/* err >= 0 after dev_alloc_name() or stores the first errno */
1127 		if (err >= 0) {
1128 			err = ret;
1129 			write_seqcount_begin(&devnet_rename_seq);
1130 			memcpy(dev->name, oldname, IFNAMSIZ);
1131 			goto rollback;
1132 		} else {
1133 			pr_err("%s: name change rollback failed: %d\n",
1134 			       dev->name, ret);
1135 		}
1136 	}
1137 
1138 	return err;
1139 }
1140 
1141 /**
1142  *	dev_set_alias - change ifalias of a device
1143  *	@dev: device
1144  *	@alias: name up to IFALIASZ
1145  *	@len: limit of bytes to copy from info
1146  *
1147  *	Set ifalias for a device,
1148  */
dev_set_alias(struct net_device * dev,const char * alias,size_t len)1149 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1150 {
1151 	char *new_ifalias;
1152 
1153 	ASSERT_RTNL();
1154 
1155 	if (len >= IFALIASZ)
1156 		return -EINVAL;
1157 
1158 	if (!len) {
1159 		kfree(dev->ifalias);
1160 		dev->ifalias = NULL;
1161 		return 0;
1162 	}
1163 
1164 	new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1165 	if (!new_ifalias)
1166 		return -ENOMEM;
1167 	dev->ifalias = new_ifalias;
1168 
1169 	strlcpy(dev->ifalias, alias, len+1);
1170 	return len;
1171 }
1172 
1173 
1174 /**
1175  *	netdev_features_change - device changes features
1176  *	@dev: device to cause notification
1177  *
1178  *	Called to indicate a device has changed features.
1179  */
netdev_features_change(struct net_device * dev)1180 void netdev_features_change(struct net_device *dev)
1181 {
1182 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1183 }
1184 EXPORT_SYMBOL(netdev_features_change);
1185 
1186 /**
1187  *	netdev_state_change - device changes state
1188  *	@dev: device to cause notification
1189  *
1190  *	Called to indicate a device has changed state. This function calls
1191  *	the notifier chains for netdev_chain and sends a NEWLINK message
1192  *	to the routing socket.
1193  */
netdev_state_change(struct net_device * dev)1194 void netdev_state_change(struct net_device *dev)
1195 {
1196 	if (dev->flags & IFF_UP) {
1197 		call_netdevice_notifiers(NETDEV_CHANGE, dev);
1198 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1199 	}
1200 }
1201 EXPORT_SYMBOL(netdev_state_change);
1202 
1203 /**
1204  * 	netdev_notify_peers - notify network peers about existence of @dev
1205  * 	@dev: network device
1206  *
1207  * Generate traffic such that interested network peers are aware of
1208  * @dev, such as by generating a gratuitous ARP. This may be used when
1209  * a device wants to inform the rest of the network about some sort of
1210  * reconfiguration such as a failover event or virtual machine
1211  * migration.
1212  */
netdev_notify_peers(struct net_device * dev)1213 void netdev_notify_peers(struct net_device *dev)
1214 {
1215 	rtnl_lock();
1216 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1217 	rtnl_unlock();
1218 }
1219 EXPORT_SYMBOL(netdev_notify_peers);
1220 
__dev_open(struct net_device * dev)1221 static int __dev_open(struct net_device *dev)
1222 {
1223 	const struct net_device_ops *ops = dev->netdev_ops;
1224 	int ret;
1225 
1226 	ASSERT_RTNL();
1227 
1228 	if (!netif_device_present(dev))
1229 		return -ENODEV;
1230 
1231 	/* Block netpoll from trying to do any rx path servicing.
1232 	 * If we don't do this there is a chance ndo_poll_controller
1233 	 * or ndo_poll may be running while we open the device
1234 	 */
1235 	ret = netpoll_rx_disable(dev);
1236 	if (ret)
1237 		return ret;
1238 
1239 	ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1240 	ret = notifier_to_errno(ret);
1241 	if (ret)
1242 		return ret;
1243 
1244 	set_bit(__LINK_STATE_START, &dev->state);
1245 
1246 	if (ops->ndo_validate_addr)
1247 		ret = ops->ndo_validate_addr(dev);
1248 
1249 	if (!ret && ops->ndo_open)
1250 		ret = ops->ndo_open(dev);
1251 
1252 	netpoll_rx_enable(dev);
1253 
1254 	if (ret)
1255 		clear_bit(__LINK_STATE_START, &dev->state);
1256 	else {
1257 		dev->flags |= IFF_UP;
1258 		net_dmaengine_get();
1259 		dev_set_rx_mode(dev);
1260 		dev_activate(dev);
1261 		add_device_randomness(dev->dev_addr, dev->addr_len);
1262 	}
1263 
1264 	return ret;
1265 }
1266 
1267 /**
1268  *	dev_open	- prepare an interface for use.
1269  *	@dev:	device to open
1270  *
1271  *	Takes a device from down to up state. The device's private open
1272  *	function is invoked and then the multicast lists are loaded. Finally
1273  *	the device is moved into the up state and a %NETDEV_UP message is
1274  *	sent to the netdev notifier chain.
1275  *
1276  *	Calling this function on an active interface is a nop. On a failure
1277  *	a negative errno code is returned.
1278  */
dev_open(struct net_device * dev)1279 int dev_open(struct net_device *dev)
1280 {
1281 	int ret;
1282 
1283 	if (dev->flags & IFF_UP)
1284 		return 0;
1285 
1286 	ret = __dev_open(dev);
1287 	if (ret < 0)
1288 		return ret;
1289 
1290 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1291 	call_netdevice_notifiers(NETDEV_UP, dev);
1292 
1293 	return ret;
1294 }
1295 EXPORT_SYMBOL(dev_open);
1296 
__dev_close_many(struct list_head * head)1297 static int __dev_close_many(struct list_head *head)
1298 {
1299 	struct net_device *dev;
1300 
1301 	ASSERT_RTNL();
1302 	might_sleep();
1303 
1304 	list_for_each_entry(dev, head, unreg_list) {
1305 		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1306 
1307 		clear_bit(__LINK_STATE_START, &dev->state);
1308 
1309 		/* Synchronize to scheduled poll. We cannot touch poll list, it
1310 		 * can be even on different cpu. So just clear netif_running().
1311 		 *
1312 		 * dev->stop() will invoke napi_disable() on all of it's
1313 		 * napi_struct instances on this device.
1314 		 */
1315 		smp_mb__after_clear_bit(); /* Commit netif_running(). */
1316 	}
1317 
1318 	dev_deactivate_many(head);
1319 
1320 	list_for_each_entry(dev, head, unreg_list) {
1321 		const struct net_device_ops *ops = dev->netdev_ops;
1322 
1323 		/*
1324 		 *	Call the device specific close. This cannot fail.
1325 		 *	Only if device is UP
1326 		 *
1327 		 *	We allow it to be called even after a DETACH hot-plug
1328 		 *	event.
1329 		 */
1330 		if (ops->ndo_stop)
1331 			ops->ndo_stop(dev);
1332 
1333 		dev->flags &= ~IFF_UP;
1334 		net_dmaengine_put();
1335 	}
1336 
1337 	return 0;
1338 }
1339 
__dev_close(struct net_device * dev)1340 static int __dev_close(struct net_device *dev)
1341 {
1342 	int retval;
1343 	LIST_HEAD(single);
1344 
1345 	/* Temporarily disable netpoll until the interface is down */
1346 	retval = netpoll_rx_disable(dev);
1347 	if (retval)
1348 		return retval;
1349 
1350 	list_add(&dev->unreg_list, &single);
1351 	retval = __dev_close_many(&single);
1352 	list_del(&single);
1353 
1354 	netpoll_rx_enable(dev);
1355 	return retval;
1356 }
1357 
dev_close_many(struct list_head * head)1358 static int dev_close_many(struct list_head *head)
1359 {
1360 	struct net_device *dev, *tmp;
1361 	LIST_HEAD(tmp_list);
1362 
1363 	list_for_each_entry_safe(dev, tmp, head, unreg_list)
1364 		if (!(dev->flags & IFF_UP))
1365 			list_move(&dev->unreg_list, &tmp_list);
1366 
1367 	__dev_close_many(head);
1368 
1369 	list_for_each_entry(dev, head, unreg_list) {
1370 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1371 		call_netdevice_notifiers(NETDEV_DOWN, dev);
1372 	}
1373 
1374 	/* rollback_registered_many needs the complete original list */
1375 	list_splice(&tmp_list, head);
1376 	return 0;
1377 }
1378 
1379 /**
1380  *	dev_close - shutdown an interface.
1381  *	@dev: device to shutdown
1382  *
1383  *	This function moves an active device into down state. A
1384  *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1385  *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1386  *	chain.
1387  */
dev_close(struct net_device * dev)1388 int dev_close(struct net_device *dev)
1389 {
1390 	int ret = 0;
1391 	if (dev->flags & IFF_UP) {
1392 		LIST_HEAD(single);
1393 
1394 		/* Block netpoll rx while the interface is going down */
1395 		ret = netpoll_rx_disable(dev);
1396 		if (ret)
1397 			return ret;
1398 
1399 		list_add(&dev->unreg_list, &single);
1400 		dev_close_many(&single);
1401 		list_del(&single);
1402 
1403 		netpoll_rx_enable(dev);
1404 	}
1405 	return ret;
1406 }
1407 EXPORT_SYMBOL(dev_close);
1408 
1409 
1410 /**
1411  *	dev_disable_lro - disable Large Receive Offload on a device
1412  *	@dev: device
1413  *
1414  *	Disable Large Receive Offload (LRO) on a net device.  Must be
1415  *	called under RTNL.  This is needed if received packets may be
1416  *	forwarded to another interface.
1417  */
dev_disable_lro(struct net_device * dev)1418 void dev_disable_lro(struct net_device *dev)
1419 {
1420 	/*
1421 	 * If we're trying to disable lro on a vlan device
1422 	 * use the underlying physical device instead
1423 	 */
1424 	if (is_vlan_dev(dev))
1425 		dev = vlan_dev_real_dev(dev);
1426 
1427 	dev->wanted_features &= ~NETIF_F_LRO;
1428 	netdev_update_features(dev);
1429 
1430 	if (unlikely(dev->features & NETIF_F_LRO))
1431 		netdev_WARN(dev, "failed to disable LRO!\n");
1432 }
1433 EXPORT_SYMBOL(dev_disable_lro);
1434 
1435 
1436 static int dev_boot_phase = 1;
1437 
1438 /**
1439  *	register_netdevice_notifier - register a network notifier block
1440  *	@nb: notifier
1441  *
1442  *	Register a notifier to be called when network device events occur.
1443  *	The notifier passed is linked into the kernel structures and must
1444  *	not be reused until it has been unregistered. A negative errno code
1445  *	is returned on a failure.
1446  *
1447  * 	When registered all registration and up events are replayed
1448  *	to the new notifier to allow device to have a race free
1449  *	view of the network device list.
1450  */
1451 
register_netdevice_notifier(struct notifier_block * nb)1452 int register_netdevice_notifier(struct notifier_block *nb)
1453 {
1454 	struct net_device *dev;
1455 	struct net_device *last;
1456 	struct net *net;
1457 	int err;
1458 
1459 	rtnl_lock();
1460 	err = raw_notifier_chain_register(&netdev_chain, nb);
1461 	if (err)
1462 		goto unlock;
1463 	if (dev_boot_phase)
1464 		goto unlock;
1465 	for_each_net(net) {
1466 		for_each_netdev(net, dev) {
1467 			err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1468 			err = notifier_to_errno(err);
1469 			if (err)
1470 				goto rollback;
1471 
1472 			if (!(dev->flags & IFF_UP))
1473 				continue;
1474 
1475 			nb->notifier_call(nb, NETDEV_UP, dev);
1476 		}
1477 	}
1478 
1479 unlock:
1480 	rtnl_unlock();
1481 	return err;
1482 
1483 rollback:
1484 	last = dev;
1485 	for_each_net(net) {
1486 		for_each_netdev(net, dev) {
1487 			if (dev == last)
1488 				goto outroll;
1489 
1490 			if (dev->flags & IFF_UP) {
1491 				nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1492 				nb->notifier_call(nb, NETDEV_DOWN, dev);
1493 			}
1494 			nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1495 		}
1496 	}
1497 
1498 outroll:
1499 	raw_notifier_chain_unregister(&netdev_chain, nb);
1500 	goto unlock;
1501 }
1502 EXPORT_SYMBOL(register_netdevice_notifier);
1503 
1504 /**
1505  *	unregister_netdevice_notifier - unregister a network notifier block
1506  *	@nb: notifier
1507  *
1508  *	Unregister a notifier previously registered by
1509  *	register_netdevice_notifier(). The notifier is unlinked into the
1510  *	kernel structures and may then be reused. A negative errno code
1511  *	is returned on a failure.
1512  *
1513  * 	After unregistering unregister and down device events are synthesized
1514  *	for all devices on the device list to the removed notifier to remove
1515  *	the need for special case cleanup code.
1516  */
1517 
unregister_netdevice_notifier(struct notifier_block * nb)1518 int unregister_netdevice_notifier(struct notifier_block *nb)
1519 {
1520 	struct net_device *dev;
1521 	struct net *net;
1522 	int err;
1523 
1524 	rtnl_lock();
1525 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1526 	if (err)
1527 		goto unlock;
1528 
1529 	for_each_net(net) {
1530 		for_each_netdev(net, dev) {
1531 			if (dev->flags & IFF_UP) {
1532 				nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1533 				nb->notifier_call(nb, NETDEV_DOWN, dev);
1534 			}
1535 			nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1536 		}
1537 	}
1538 unlock:
1539 	rtnl_unlock();
1540 	return err;
1541 }
1542 EXPORT_SYMBOL(unregister_netdevice_notifier);
1543 
1544 /**
1545  *	call_netdevice_notifiers - call all network notifier blocks
1546  *      @val: value passed unmodified to notifier function
1547  *      @dev: net_device pointer passed unmodified to notifier function
1548  *
1549  *	Call all network notifier blocks.  Parameters and return value
1550  *	are as for raw_notifier_call_chain().
1551  */
1552 
call_netdevice_notifiers(unsigned long val,struct net_device * dev)1553 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1554 {
1555 	ASSERT_RTNL();
1556 	return raw_notifier_call_chain(&netdev_chain, val, dev);
1557 }
1558 EXPORT_SYMBOL(call_netdevice_notifiers);
1559 
1560 static struct static_key netstamp_needed __read_mostly;
1561 #ifdef HAVE_JUMP_LABEL
1562 /* We are not allowed to call static_key_slow_dec() from irq context
1563  * If net_disable_timestamp() is called from irq context, defer the
1564  * static_key_slow_dec() calls.
1565  */
1566 static atomic_t netstamp_needed_deferred;
1567 #endif
1568 
net_enable_timestamp(void)1569 void net_enable_timestamp(void)
1570 {
1571 #ifdef HAVE_JUMP_LABEL
1572 	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1573 
1574 	if (deferred) {
1575 		while (--deferred)
1576 			static_key_slow_dec(&netstamp_needed);
1577 		return;
1578 	}
1579 #endif
1580 	static_key_slow_inc(&netstamp_needed);
1581 }
1582 EXPORT_SYMBOL(net_enable_timestamp);
1583 
net_disable_timestamp(void)1584 void net_disable_timestamp(void)
1585 {
1586 #ifdef HAVE_JUMP_LABEL
1587 	if (in_interrupt()) {
1588 		atomic_inc(&netstamp_needed_deferred);
1589 		return;
1590 	}
1591 #endif
1592 	static_key_slow_dec(&netstamp_needed);
1593 }
1594 EXPORT_SYMBOL(net_disable_timestamp);
1595 
net_timestamp_set(struct sk_buff * skb)1596 static inline void net_timestamp_set(struct sk_buff *skb)
1597 {
1598 	skb->tstamp.tv64 = 0;
1599 	if (static_key_false(&netstamp_needed))
1600 		__net_timestamp(skb);
1601 }
1602 
1603 #define net_timestamp_check(COND, SKB)			\
1604 	if (static_key_false(&netstamp_needed)) {		\
1605 		if ((COND) && !(SKB)->tstamp.tv64)	\
1606 			__net_timestamp(SKB);		\
1607 	}						\
1608 
is_skb_forwardable(struct net_device * dev,struct sk_buff * skb)1609 static inline bool is_skb_forwardable(struct net_device *dev,
1610 				      struct sk_buff *skb)
1611 {
1612 	unsigned int len;
1613 
1614 	if (!(dev->flags & IFF_UP))
1615 		return false;
1616 
1617 	len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1618 	if (skb->len <= len)
1619 		return true;
1620 
1621 	/* if TSO is enabled, we don't care about the length as the packet
1622 	 * could be forwarded without being segmented before
1623 	 */
1624 	if (skb_is_gso(skb))
1625 		return true;
1626 
1627 	return false;
1628 }
1629 
1630 /**
1631  * dev_forward_skb - loopback an skb to another netif
1632  *
1633  * @dev: destination network device
1634  * @skb: buffer to forward
1635  *
1636  * return values:
1637  *	NET_RX_SUCCESS	(no congestion)
1638  *	NET_RX_DROP     (packet was dropped, but freed)
1639  *
1640  * dev_forward_skb can be used for injecting an skb from the
1641  * start_xmit function of one device into the receive queue
1642  * of another device.
1643  *
1644  * The receiving device may be in another namespace, so
1645  * we have to clear all information in the skb that could
1646  * impact namespace isolation.
1647  */
dev_forward_skb(struct net_device * dev,struct sk_buff * skb)1648 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1649 {
1650 	if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1651 		if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1652 			atomic_long_inc(&dev->rx_dropped);
1653 			kfree_skb(skb);
1654 			return NET_RX_DROP;
1655 		}
1656 	}
1657 
1658 	skb_orphan(skb);
1659 
1660 	if (unlikely(!is_skb_forwardable(dev, skb))) {
1661 		atomic_long_inc(&dev->rx_dropped);
1662 		kfree_skb(skb);
1663 		return NET_RX_DROP;
1664 	}
1665 	skb->skb_iif = 0;
1666 	skb->dev = dev;
1667 	skb_dst_drop(skb);
1668 	skb->tstamp.tv64 = 0;
1669 	skb->pkt_type = PACKET_HOST;
1670 	skb->protocol = eth_type_trans(skb, dev);
1671 	skb->mark = 0;
1672 	secpath_reset(skb);
1673 	nf_reset(skb);
1674 	nf_reset_trace(skb);
1675 	return netif_rx(skb);
1676 }
1677 EXPORT_SYMBOL_GPL(dev_forward_skb);
1678 
deliver_skb(struct sk_buff * skb,struct packet_type * pt_prev,struct net_device * orig_dev)1679 static inline int deliver_skb(struct sk_buff *skb,
1680 			      struct packet_type *pt_prev,
1681 			      struct net_device *orig_dev)
1682 {
1683 	if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1684 		return -ENOMEM;
1685 	atomic_inc(&skb->users);
1686 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1687 }
1688 
skb_loop_sk(struct packet_type * ptype,struct sk_buff * skb)1689 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1690 {
1691 	if (!ptype->af_packet_priv || !skb->sk)
1692 		return false;
1693 
1694 	if (ptype->id_match)
1695 		return ptype->id_match(ptype, skb->sk);
1696 	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1697 		return true;
1698 
1699 	return false;
1700 }
1701 
1702 /*
1703  *	Support routine. Sends outgoing frames to any network
1704  *	taps currently in use.
1705  */
1706 
dev_queue_xmit_nit(struct sk_buff * skb,struct net_device * dev)1707 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1708 {
1709 	struct packet_type *ptype;
1710 	struct sk_buff *skb2 = NULL;
1711 	struct packet_type *pt_prev = NULL;
1712 
1713 	rcu_read_lock();
1714 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
1715 		/* Never send packets back to the socket
1716 		 * they originated from - MvS (miquels@drinkel.ow.org)
1717 		 */
1718 		if ((ptype->dev == dev || !ptype->dev) &&
1719 		    (!skb_loop_sk(ptype, skb))) {
1720 			if (pt_prev) {
1721 				deliver_skb(skb2, pt_prev, skb->dev);
1722 				pt_prev = ptype;
1723 				continue;
1724 			}
1725 
1726 			skb2 = skb_clone(skb, GFP_ATOMIC);
1727 			if (!skb2)
1728 				break;
1729 
1730 			net_timestamp_set(skb2);
1731 
1732 			/* skb->nh should be correctly
1733 			   set by sender, so that the second statement is
1734 			   just protection against buggy protocols.
1735 			 */
1736 			skb_reset_mac_header(skb2);
1737 
1738 			if (skb_network_header(skb2) < skb2->data ||
1739 			    skb2->network_header > skb2->tail) {
1740 				net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1741 						     ntohs(skb2->protocol),
1742 						     dev->name);
1743 				skb_reset_network_header(skb2);
1744 			}
1745 
1746 			skb2->transport_header = skb2->network_header;
1747 			skb2->pkt_type = PACKET_OUTGOING;
1748 			pt_prev = ptype;
1749 		}
1750 	}
1751 	if (pt_prev)
1752 		pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1753 	rcu_read_unlock();
1754 }
1755 
1756 /**
1757  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1758  * @dev: Network device
1759  * @txq: number of queues available
1760  *
1761  * If real_num_tx_queues is changed the tc mappings may no longer be
1762  * valid. To resolve this verify the tc mapping remains valid and if
1763  * not NULL the mapping. With no priorities mapping to this
1764  * offset/count pair it will no longer be used. In the worst case TC0
1765  * is invalid nothing can be done so disable priority mappings. If is
1766  * expected that drivers will fix this mapping if they can before
1767  * calling netif_set_real_num_tx_queues.
1768  */
netif_setup_tc(struct net_device * dev,unsigned int txq)1769 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1770 {
1771 	int i;
1772 	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1773 
1774 	/* If TC0 is invalidated disable TC mapping */
1775 	if (tc->offset + tc->count > txq) {
1776 		pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1777 		dev->num_tc = 0;
1778 		return;
1779 	}
1780 
1781 	/* Invalidated prio to tc mappings set to TC0 */
1782 	for (i = 1; i < TC_BITMASK + 1; i++) {
1783 		int q = netdev_get_prio_tc_map(dev, i);
1784 
1785 		tc = &dev->tc_to_txq[q];
1786 		if (tc->offset + tc->count > txq) {
1787 			pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1788 				i, q);
1789 			netdev_set_prio_tc_map(dev, i, 0);
1790 		}
1791 	}
1792 }
1793 
1794 #ifdef CONFIG_XPS
1795 static DEFINE_MUTEX(xps_map_mutex);
1796 #define xmap_dereference(P)		\
1797 	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1798 
remove_xps_queue(struct xps_dev_maps * dev_maps,int cpu,u16 index)1799 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1800 					int cpu, u16 index)
1801 {
1802 	struct xps_map *map = NULL;
1803 	int pos;
1804 
1805 	if (dev_maps)
1806 		map = xmap_dereference(dev_maps->cpu_map[cpu]);
1807 
1808 	for (pos = 0; map && pos < map->len; pos++) {
1809 		if (map->queues[pos] == index) {
1810 			if (map->len > 1) {
1811 				map->queues[pos] = map->queues[--map->len];
1812 			} else {
1813 				RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1814 				kfree_rcu(map, rcu);
1815 				map = NULL;
1816 			}
1817 			break;
1818 		}
1819 	}
1820 
1821 	return map;
1822 }
1823 
netif_reset_xps_queues_gt(struct net_device * dev,u16 index)1824 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1825 {
1826 	struct xps_dev_maps *dev_maps;
1827 	int cpu, i;
1828 	bool active = false;
1829 
1830 	mutex_lock(&xps_map_mutex);
1831 	dev_maps = xmap_dereference(dev->xps_maps);
1832 
1833 	if (!dev_maps)
1834 		goto out_no_maps;
1835 
1836 	for_each_possible_cpu(cpu) {
1837 		for (i = index; i < dev->num_tx_queues; i++) {
1838 			if (!remove_xps_queue(dev_maps, cpu, i))
1839 				break;
1840 		}
1841 		if (i == dev->num_tx_queues)
1842 			active = true;
1843 	}
1844 
1845 	if (!active) {
1846 		RCU_INIT_POINTER(dev->xps_maps, NULL);
1847 		kfree_rcu(dev_maps, rcu);
1848 	}
1849 
1850 	for (i = index; i < dev->num_tx_queues; i++)
1851 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1852 					     NUMA_NO_NODE);
1853 
1854 out_no_maps:
1855 	mutex_unlock(&xps_map_mutex);
1856 }
1857 
expand_xps_map(struct xps_map * map,int cpu,u16 index)1858 static struct xps_map *expand_xps_map(struct xps_map *map,
1859 				      int cpu, u16 index)
1860 {
1861 	struct xps_map *new_map;
1862 	int alloc_len = XPS_MIN_MAP_ALLOC;
1863 	int i, pos;
1864 
1865 	for (pos = 0; map && pos < map->len; pos++) {
1866 		if (map->queues[pos] != index)
1867 			continue;
1868 		return map;
1869 	}
1870 
1871 	/* Need to add queue to this CPU's existing map */
1872 	if (map) {
1873 		if (pos < map->alloc_len)
1874 			return map;
1875 
1876 		alloc_len = map->alloc_len * 2;
1877 	}
1878 
1879 	/* Need to allocate new map to store queue on this CPU's map */
1880 	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1881 			       cpu_to_node(cpu));
1882 	if (!new_map)
1883 		return NULL;
1884 
1885 	for (i = 0; i < pos; i++)
1886 		new_map->queues[i] = map->queues[i];
1887 	new_map->alloc_len = alloc_len;
1888 	new_map->len = pos;
1889 
1890 	return new_map;
1891 }
1892 
netif_set_xps_queue(struct net_device * dev,struct cpumask * mask,u16 index)1893 int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)
1894 {
1895 	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
1896 	struct xps_map *map, *new_map;
1897 	int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
1898 	int cpu, numa_node_id = -2;
1899 	bool active = false;
1900 
1901 	mutex_lock(&xps_map_mutex);
1902 
1903 	dev_maps = xmap_dereference(dev->xps_maps);
1904 
1905 	/* allocate memory for queue storage */
1906 	for_each_online_cpu(cpu) {
1907 		if (!cpumask_test_cpu(cpu, mask))
1908 			continue;
1909 
1910 		if (!new_dev_maps)
1911 			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
1912 		if (!new_dev_maps) {
1913 			mutex_unlock(&xps_map_mutex);
1914 			return -ENOMEM;
1915 		}
1916 
1917 		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
1918 				 NULL;
1919 
1920 		map = expand_xps_map(map, cpu, index);
1921 		if (!map)
1922 			goto error;
1923 
1924 		RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1925 	}
1926 
1927 	if (!new_dev_maps)
1928 		goto out_no_new_maps;
1929 
1930 	for_each_possible_cpu(cpu) {
1931 		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
1932 			/* add queue to CPU maps */
1933 			int pos = 0;
1934 
1935 			map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1936 			while ((pos < map->len) && (map->queues[pos] != index))
1937 				pos++;
1938 
1939 			if (pos == map->len)
1940 				map->queues[map->len++] = index;
1941 #ifdef CONFIG_NUMA
1942 			if (numa_node_id == -2)
1943 				numa_node_id = cpu_to_node(cpu);
1944 			else if (numa_node_id != cpu_to_node(cpu))
1945 				numa_node_id = -1;
1946 #endif
1947 		} else if (dev_maps) {
1948 			/* fill in the new device map from the old device map */
1949 			map = xmap_dereference(dev_maps->cpu_map[cpu]);
1950 			RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1951 		}
1952 
1953 	}
1954 
1955 	rcu_assign_pointer(dev->xps_maps, new_dev_maps);
1956 
1957 	/* Cleanup old maps */
1958 	if (dev_maps) {
1959 		for_each_possible_cpu(cpu) {
1960 			new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1961 			map = xmap_dereference(dev_maps->cpu_map[cpu]);
1962 			if (map && map != new_map)
1963 				kfree_rcu(map, rcu);
1964 		}
1965 
1966 		kfree_rcu(dev_maps, rcu);
1967 	}
1968 
1969 	dev_maps = new_dev_maps;
1970 	active = true;
1971 
1972 out_no_new_maps:
1973 	/* update Tx queue numa node */
1974 	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
1975 				     (numa_node_id >= 0) ? numa_node_id :
1976 				     NUMA_NO_NODE);
1977 
1978 	if (!dev_maps)
1979 		goto out_no_maps;
1980 
1981 	/* removes queue from unused CPUs */
1982 	for_each_possible_cpu(cpu) {
1983 		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
1984 			continue;
1985 
1986 		if (remove_xps_queue(dev_maps, cpu, index))
1987 			active = true;
1988 	}
1989 
1990 	/* free map if not active */
1991 	if (!active) {
1992 		RCU_INIT_POINTER(dev->xps_maps, NULL);
1993 		kfree_rcu(dev_maps, rcu);
1994 	}
1995 
1996 out_no_maps:
1997 	mutex_unlock(&xps_map_mutex);
1998 
1999 	return 0;
2000 error:
2001 	/* remove any maps that we added */
2002 	for_each_possible_cpu(cpu) {
2003 		new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2004 		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2005 				 NULL;
2006 		if (new_map && new_map != map)
2007 			kfree(new_map);
2008 	}
2009 
2010 	mutex_unlock(&xps_map_mutex);
2011 
2012 	kfree(new_dev_maps);
2013 	return -ENOMEM;
2014 }
2015 EXPORT_SYMBOL(netif_set_xps_queue);
2016 
2017 #endif
2018 /*
2019  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2020  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2021  */
netif_set_real_num_tx_queues(struct net_device * dev,unsigned int txq)2022 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2023 {
2024 	int rc;
2025 
2026 	if (txq < 1 || txq > dev->num_tx_queues)
2027 		return -EINVAL;
2028 
2029 	if (dev->reg_state == NETREG_REGISTERED ||
2030 	    dev->reg_state == NETREG_UNREGISTERING) {
2031 		ASSERT_RTNL();
2032 
2033 		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2034 						  txq);
2035 		if (rc)
2036 			return rc;
2037 
2038 		if (dev->num_tc)
2039 			netif_setup_tc(dev, txq);
2040 
2041 		if (txq < dev->real_num_tx_queues) {
2042 			qdisc_reset_all_tx_gt(dev, txq);
2043 #ifdef CONFIG_XPS
2044 			netif_reset_xps_queues_gt(dev, txq);
2045 #endif
2046 		}
2047 	}
2048 
2049 	dev->real_num_tx_queues = txq;
2050 	return 0;
2051 }
2052 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2053 
2054 #ifdef CONFIG_RPS
2055 /**
2056  *	netif_set_real_num_rx_queues - set actual number of RX queues used
2057  *	@dev: Network device
2058  *	@rxq: Actual number of RX queues
2059  *
2060  *	This must be called either with the rtnl_lock held or before
2061  *	registration of the net device.  Returns 0 on success, or a
2062  *	negative error code.  If called before registration, it always
2063  *	succeeds.
2064  */
netif_set_real_num_rx_queues(struct net_device * dev,unsigned int rxq)2065 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2066 {
2067 	int rc;
2068 
2069 	if (rxq < 1 || rxq > dev->num_rx_queues)
2070 		return -EINVAL;
2071 
2072 	if (dev->reg_state == NETREG_REGISTERED) {
2073 		ASSERT_RTNL();
2074 
2075 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2076 						  rxq);
2077 		if (rc)
2078 			return rc;
2079 	}
2080 
2081 	dev->real_num_rx_queues = rxq;
2082 	return 0;
2083 }
2084 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2085 #endif
2086 
2087 /**
2088  * netif_get_num_default_rss_queues - default number of RSS queues
2089  *
2090  * This routine should set an upper limit on the number of RSS queues
2091  * used by default by multiqueue devices.
2092  */
netif_get_num_default_rss_queues(void)2093 int netif_get_num_default_rss_queues(void)
2094 {
2095 	return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2096 }
2097 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2098 
__netif_reschedule(struct Qdisc * q)2099 static inline void __netif_reschedule(struct Qdisc *q)
2100 {
2101 	struct softnet_data *sd;
2102 	unsigned long flags;
2103 
2104 	local_irq_save(flags);
2105 	sd = &__get_cpu_var(softnet_data);
2106 	q->next_sched = NULL;
2107 	*sd->output_queue_tailp = q;
2108 	sd->output_queue_tailp = &q->next_sched;
2109 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2110 	local_irq_restore(flags);
2111 }
2112 
__netif_schedule(struct Qdisc * q)2113 void __netif_schedule(struct Qdisc *q)
2114 {
2115 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2116 		__netif_reschedule(q);
2117 }
2118 EXPORT_SYMBOL(__netif_schedule);
2119 
dev_kfree_skb_irq(struct sk_buff * skb)2120 void dev_kfree_skb_irq(struct sk_buff *skb)
2121 {
2122 	if (atomic_dec_and_test(&skb->users)) {
2123 		struct softnet_data *sd;
2124 		unsigned long flags;
2125 
2126 		local_irq_save(flags);
2127 		sd = &__get_cpu_var(softnet_data);
2128 		skb->next = sd->completion_queue;
2129 		sd->completion_queue = skb;
2130 		raise_softirq_irqoff(NET_TX_SOFTIRQ);
2131 		local_irq_restore(flags);
2132 	}
2133 }
2134 EXPORT_SYMBOL(dev_kfree_skb_irq);
2135 
dev_kfree_skb_any(struct sk_buff * skb)2136 void dev_kfree_skb_any(struct sk_buff *skb)
2137 {
2138 	if (in_irq() || irqs_disabled())
2139 		dev_kfree_skb_irq(skb);
2140 	else
2141 		dev_kfree_skb(skb);
2142 }
2143 EXPORT_SYMBOL(dev_kfree_skb_any);
2144 
2145 
2146 /**
2147  * netif_device_detach - mark device as removed
2148  * @dev: network device
2149  *
2150  * Mark device as removed from system and therefore no longer available.
2151  */
netif_device_detach(struct net_device * dev)2152 void netif_device_detach(struct net_device *dev)
2153 {
2154 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2155 	    netif_running(dev)) {
2156 		netif_tx_stop_all_queues(dev);
2157 	}
2158 }
2159 EXPORT_SYMBOL(netif_device_detach);
2160 
2161 /**
2162  * netif_device_attach - mark device as attached
2163  * @dev: network device
2164  *
2165  * Mark device as attached from system and restart if needed.
2166  */
netif_device_attach(struct net_device * dev)2167 void netif_device_attach(struct net_device *dev)
2168 {
2169 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2170 	    netif_running(dev)) {
2171 		netif_tx_wake_all_queues(dev);
2172 		__netdev_watchdog_up(dev);
2173 	}
2174 }
2175 EXPORT_SYMBOL(netif_device_attach);
2176 
skb_warn_bad_offload(const struct sk_buff * skb)2177 static void skb_warn_bad_offload(const struct sk_buff *skb)
2178 {
2179 	static const netdev_features_t null_features = 0;
2180 	struct net_device *dev = skb->dev;
2181 	const char *driver = "";
2182 
2183 	if (!net_ratelimit())
2184 		return;
2185 
2186 	if (dev && dev->dev.parent)
2187 		driver = dev_driver_string(dev->dev.parent);
2188 
2189 	WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2190 	     "gso_type=%d ip_summed=%d\n",
2191 	     driver, dev ? &dev->features : &null_features,
2192 	     skb->sk ? &skb->sk->sk_route_caps : &null_features,
2193 	     skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2194 	     skb_shinfo(skb)->gso_type, skb->ip_summed);
2195 }
2196 
2197 /*
2198  * Invalidate hardware checksum when packet is to be mangled, and
2199  * complete checksum manually on outgoing path.
2200  */
skb_checksum_help(struct sk_buff * skb)2201 int skb_checksum_help(struct sk_buff *skb)
2202 {
2203 	__wsum csum;
2204 	int ret = 0, offset;
2205 
2206 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2207 		goto out_set_summed;
2208 
2209 	if (unlikely(skb_shinfo(skb)->gso_size)) {
2210 		skb_warn_bad_offload(skb);
2211 		return -EINVAL;
2212 	}
2213 
2214 	/* Before computing a checksum, we should make sure no frag could
2215 	 * be modified by an external entity : checksum could be wrong.
2216 	 */
2217 	if (skb_has_shared_frag(skb)) {
2218 		ret = __skb_linearize(skb);
2219 		if (ret)
2220 			goto out;
2221 	}
2222 
2223 	offset = skb_checksum_start_offset(skb);
2224 	BUG_ON(offset >= skb_headlen(skb));
2225 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
2226 
2227 	offset += skb->csum_offset;
2228 	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2229 
2230 	if (skb_cloned(skb) &&
2231 	    !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2232 		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2233 		if (ret)
2234 			goto out;
2235 	}
2236 
2237 	*(__sum16 *)(skb->data + offset) = csum_fold(csum);
2238 out_set_summed:
2239 	skb->ip_summed = CHECKSUM_NONE;
2240 out:
2241 	return ret;
2242 }
2243 EXPORT_SYMBOL(skb_checksum_help);
2244 
skb_network_protocol(struct sk_buff * skb)2245 __be16 skb_network_protocol(struct sk_buff *skb)
2246 {
2247 	__be16 type = skb->protocol;
2248 	int vlan_depth = ETH_HLEN;
2249 
2250 	/* Tunnel gso handlers can set protocol to ethernet. */
2251 	if (type == htons(ETH_P_TEB)) {
2252 		struct ethhdr *eth;
2253 
2254 		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2255 			return 0;
2256 
2257 		eth = (struct ethhdr *)skb_mac_header(skb);
2258 		type = eth->h_proto;
2259 	}
2260 
2261 	while (type == htons(ETH_P_8021Q) || type == htons(ETH_P_8021AD)) {
2262 		struct vlan_hdr *vh;
2263 
2264 		if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
2265 			return 0;
2266 
2267 		vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2268 		type = vh->h_vlan_encapsulated_proto;
2269 		vlan_depth += VLAN_HLEN;
2270 	}
2271 
2272 	return type;
2273 }
2274 
2275 /**
2276  *	skb_mac_gso_segment - mac layer segmentation handler.
2277  *	@skb: buffer to segment
2278  *	@features: features for the output path (see dev->features)
2279  */
skb_mac_gso_segment(struct sk_buff * skb,netdev_features_t features)2280 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2281 				    netdev_features_t features)
2282 {
2283 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2284 	struct packet_offload *ptype;
2285 	__be16 type = skb_network_protocol(skb);
2286 
2287 	if (unlikely(!type))
2288 		return ERR_PTR(-EINVAL);
2289 
2290 	__skb_pull(skb, skb->mac_len);
2291 
2292 	rcu_read_lock();
2293 	list_for_each_entry_rcu(ptype, &offload_base, list) {
2294 		if (ptype->type == type && ptype->callbacks.gso_segment) {
2295 			if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
2296 				int err;
2297 
2298 				err = ptype->callbacks.gso_send_check(skb);
2299 				segs = ERR_PTR(err);
2300 				if (err || skb_gso_ok(skb, features))
2301 					break;
2302 				__skb_push(skb, (skb->data -
2303 						 skb_network_header(skb)));
2304 			}
2305 			segs = ptype->callbacks.gso_segment(skb, features);
2306 			break;
2307 		}
2308 	}
2309 	rcu_read_unlock();
2310 
2311 	__skb_push(skb, skb->data - skb_mac_header(skb));
2312 
2313 	return segs;
2314 }
2315 EXPORT_SYMBOL(skb_mac_gso_segment);
2316 
2317 
2318 /* openvswitch calls this on rx path, so we need a different check.
2319  */
skb_needs_check(struct sk_buff * skb,bool tx_path)2320 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2321 {
2322 	if (tx_path)
2323 		return skb->ip_summed != CHECKSUM_PARTIAL;
2324 	else
2325 		return skb->ip_summed == CHECKSUM_NONE;
2326 }
2327 
2328 /**
2329  *	__skb_gso_segment - Perform segmentation on skb.
2330  *	@skb: buffer to segment
2331  *	@features: features for the output path (see dev->features)
2332  *	@tx_path: whether it is called in TX path
2333  *
2334  *	This function segments the given skb and returns a list of segments.
2335  *
2336  *	It may return NULL if the skb requires no segmentation.  This is
2337  *	only possible when GSO is used for verifying header integrity.
2338  */
__skb_gso_segment(struct sk_buff * skb,netdev_features_t features,bool tx_path)2339 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2340 				  netdev_features_t features, bool tx_path)
2341 {
2342 	if (unlikely(skb_needs_check(skb, tx_path))) {
2343 		int err;
2344 
2345 		skb_warn_bad_offload(skb);
2346 
2347 		if (skb_header_cloned(skb) &&
2348 		    (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2349 			return ERR_PTR(err);
2350 	}
2351 
2352 	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2353 	skb_reset_mac_header(skb);
2354 	skb_reset_mac_len(skb);
2355 
2356 	return skb_mac_gso_segment(skb, features);
2357 }
2358 EXPORT_SYMBOL(__skb_gso_segment);
2359 
2360 /* Take action when hardware reception checksum errors are detected. */
2361 #ifdef CONFIG_BUG
netdev_rx_csum_fault(struct net_device * dev)2362 void netdev_rx_csum_fault(struct net_device *dev)
2363 {
2364 	if (net_ratelimit()) {
2365 		pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2366 		dump_stack();
2367 	}
2368 }
2369 EXPORT_SYMBOL(netdev_rx_csum_fault);
2370 #endif
2371 
2372 /* Actually, we should eliminate this check as soon as we know, that:
2373  * 1. IOMMU is present and allows to map all the memory.
2374  * 2. No high memory really exists on this machine.
2375  */
2376 
illegal_highdma(struct net_device * dev,struct sk_buff * skb)2377 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2378 {
2379 #ifdef CONFIG_HIGHMEM
2380 	int i;
2381 	if (!(dev->features & NETIF_F_HIGHDMA)) {
2382 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2383 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2384 			if (PageHighMem(skb_frag_page(frag)))
2385 				return 1;
2386 		}
2387 	}
2388 
2389 	if (PCI_DMA_BUS_IS_PHYS) {
2390 		struct device *pdev = dev->dev.parent;
2391 
2392 		if (!pdev)
2393 			return 0;
2394 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2395 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2396 			dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2397 			if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2398 				return 1;
2399 		}
2400 	}
2401 #endif
2402 	return 0;
2403 }
2404 
2405 struct dev_gso_cb {
2406 	void (*destructor)(struct sk_buff *skb);
2407 };
2408 
2409 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2410 
dev_gso_skb_destructor(struct sk_buff * skb)2411 static void dev_gso_skb_destructor(struct sk_buff *skb)
2412 {
2413 	struct dev_gso_cb *cb;
2414 
2415 	do {
2416 		struct sk_buff *nskb = skb->next;
2417 
2418 		skb->next = nskb->next;
2419 		nskb->next = NULL;
2420 		kfree_skb(nskb);
2421 	} while (skb->next);
2422 
2423 	cb = DEV_GSO_CB(skb);
2424 	if (cb->destructor)
2425 		cb->destructor(skb);
2426 }
2427 
2428 /**
2429  *	dev_gso_segment - Perform emulated hardware segmentation on skb.
2430  *	@skb: buffer to segment
2431  *	@features: device features as applicable to this skb
2432  *
2433  *	This function segments the given skb and stores the list of segments
2434  *	in skb->next.
2435  */
dev_gso_segment(struct sk_buff * skb,netdev_features_t features)2436 static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
2437 {
2438 	struct sk_buff *segs;
2439 
2440 	segs = skb_gso_segment(skb, features);
2441 
2442 	/* Verifying header integrity only. */
2443 	if (!segs)
2444 		return 0;
2445 
2446 	if (IS_ERR(segs))
2447 		return PTR_ERR(segs);
2448 
2449 	skb->next = segs;
2450 	DEV_GSO_CB(skb)->destructor = skb->destructor;
2451 	skb->destructor = dev_gso_skb_destructor;
2452 
2453 	return 0;
2454 }
2455 
harmonize_features(struct sk_buff * skb,__be16 protocol,netdev_features_t features)2456 static netdev_features_t harmonize_features(struct sk_buff *skb,
2457 	__be16 protocol, netdev_features_t features)
2458 {
2459 	if (skb->ip_summed != CHECKSUM_NONE &&
2460 	    !can_checksum_protocol(features, protocol)) {
2461 		features &= ~NETIF_F_ALL_CSUM;
2462 	} else if (illegal_highdma(skb->dev, skb)) {
2463 		features &= ~NETIF_F_SG;
2464 	}
2465 
2466 	return features;
2467 }
2468 
netif_skb_features(struct sk_buff * skb)2469 netdev_features_t netif_skb_features(struct sk_buff *skb)
2470 {
2471 	__be16 protocol = skb->protocol;
2472 	netdev_features_t features = skb->dev->features;
2473 
2474 	if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2475 		features &= ~NETIF_F_GSO_MASK;
2476 
2477 	if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD)) {
2478 		struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2479 		protocol = veh->h_vlan_encapsulated_proto;
2480 	} else if (!vlan_tx_tag_present(skb)) {
2481 		return harmonize_features(skb, protocol, features);
2482 	}
2483 
2484 	features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_CTAG_TX |
2485 					       NETIF_F_HW_VLAN_STAG_TX);
2486 
2487 	if (protocol != htons(ETH_P_8021Q) && protocol != htons(ETH_P_8021AD)) {
2488 		return harmonize_features(skb, protocol, features);
2489 	} else {
2490 		features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
2491 				NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2492 				NETIF_F_HW_VLAN_STAG_TX;
2493 		return harmonize_features(skb, protocol, features);
2494 	}
2495 }
2496 EXPORT_SYMBOL(netif_skb_features);
2497 
2498 /*
2499  * Returns true if either:
2500  *	1. skb has frag_list and the device doesn't support FRAGLIST, or
2501  *	2. skb is fragmented and the device does not support SG.
2502  */
skb_needs_linearize(struct sk_buff * skb,netdev_features_t features)2503 static inline int skb_needs_linearize(struct sk_buff *skb,
2504 				      netdev_features_t features)
2505 {
2506 	return skb_is_nonlinear(skb) &&
2507 			((skb_has_frag_list(skb) &&
2508 				!(features & NETIF_F_FRAGLIST)) ||
2509 			(skb_shinfo(skb)->nr_frags &&
2510 				!(features & NETIF_F_SG)));
2511 }
2512 
dev_hard_start_xmit(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq)2513 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2514 			struct netdev_queue *txq)
2515 {
2516 	const struct net_device_ops *ops = dev->netdev_ops;
2517 	int rc = NETDEV_TX_OK;
2518 	unsigned int skb_len;
2519 
2520 	if (likely(!skb->next)) {
2521 		netdev_features_t features;
2522 
2523 		/*
2524 		 * If device doesn't need skb->dst, release it right now while
2525 		 * its hot in this cpu cache
2526 		 */
2527 		if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2528 			skb_dst_drop(skb);
2529 
2530 		features = netif_skb_features(skb);
2531 
2532 		if (vlan_tx_tag_present(skb) &&
2533 		    !vlan_hw_offload_capable(features, skb->vlan_proto)) {
2534 			skb = __vlan_put_tag(skb, skb->vlan_proto,
2535 					     vlan_tx_tag_get(skb));
2536 			if (unlikely(!skb))
2537 				goto out;
2538 
2539 			skb->vlan_tci = 0;
2540 		}
2541 
2542 		/* If encapsulation offload request, verify we are testing
2543 		 * hardware encapsulation features instead of standard
2544 		 * features for the netdev
2545 		 */
2546 		if (skb->encapsulation)
2547 			features &= dev->hw_enc_features;
2548 
2549 		if (netif_needs_gso(skb, features)) {
2550 			if (unlikely(dev_gso_segment(skb, features)))
2551 				goto out_kfree_skb;
2552 			if (skb->next)
2553 				goto gso;
2554 		} else {
2555 			if (skb_needs_linearize(skb, features) &&
2556 			    __skb_linearize(skb))
2557 				goto out_kfree_skb;
2558 
2559 			/* If packet is not checksummed and device does not
2560 			 * support checksumming for this protocol, complete
2561 			 * checksumming here.
2562 			 */
2563 			if (skb->ip_summed == CHECKSUM_PARTIAL) {
2564 				if (skb->encapsulation)
2565 					skb_set_inner_transport_header(skb,
2566 						skb_checksum_start_offset(skb));
2567 				else
2568 					skb_set_transport_header(skb,
2569 						skb_checksum_start_offset(skb));
2570 				if (!(features & NETIF_F_ALL_CSUM) &&
2571 				     skb_checksum_help(skb))
2572 					goto out_kfree_skb;
2573 			}
2574 		}
2575 
2576 		if (!list_empty(&ptype_all))
2577 			dev_queue_xmit_nit(skb, dev);
2578 
2579 		skb_len = skb->len;
2580 		rc = ops->ndo_start_xmit(skb, dev);
2581 		trace_net_dev_xmit(skb, rc, dev, skb_len);
2582 		if (rc == NETDEV_TX_OK)
2583 			txq_trans_update(txq);
2584 		return rc;
2585 	}
2586 
2587 gso:
2588 	do {
2589 		struct sk_buff *nskb = skb->next;
2590 
2591 		skb->next = nskb->next;
2592 		nskb->next = NULL;
2593 
2594 		if (!list_empty(&ptype_all))
2595 			dev_queue_xmit_nit(nskb, dev);
2596 
2597 		skb_len = nskb->len;
2598 		rc = ops->ndo_start_xmit(nskb, dev);
2599 		trace_net_dev_xmit(nskb, rc, dev, skb_len);
2600 		if (unlikely(rc != NETDEV_TX_OK)) {
2601 			if (rc & ~NETDEV_TX_MASK)
2602 				goto out_kfree_gso_skb;
2603 			nskb->next = skb->next;
2604 			skb->next = nskb;
2605 			return rc;
2606 		}
2607 		txq_trans_update(txq);
2608 		if (unlikely(netif_xmit_stopped(txq) && skb->next))
2609 			return NETDEV_TX_BUSY;
2610 	} while (skb->next);
2611 
2612 out_kfree_gso_skb:
2613 	if (likely(skb->next == NULL)) {
2614 		skb->destructor = DEV_GSO_CB(skb)->destructor;
2615 		consume_skb(skb);
2616 		return rc;
2617 	}
2618 out_kfree_skb:
2619 	kfree_skb(skb);
2620 out:
2621 	return rc;
2622 }
2623 
qdisc_pkt_len_init(struct sk_buff * skb)2624 static void qdisc_pkt_len_init(struct sk_buff *skb)
2625 {
2626 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
2627 
2628 	qdisc_skb_cb(skb)->pkt_len = skb->len;
2629 
2630 	/* To get more precise estimation of bytes sent on wire,
2631 	 * we add to pkt_len the headers size of all segments
2632 	 */
2633 	if (shinfo->gso_size)  {
2634 		unsigned int hdr_len;
2635 		u16 gso_segs = shinfo->gso_segs;
2636 
2637 		/* mac layer + network layer */
2638 		hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2639 
2640 		/* + transport layer */
2641 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2642 			hdr_len += tcp_hdrlen(skb);
2643 		else
2644 			hdr_len += sizeof(struct udphdr);
2645 
2646 		if (shinfo->gso_type & SKB_GSO_DODGY)
2647 			gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2648 						shinfo->gso_size);
2649 
2650 		qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
2651 	}
2652 }
2653 
__dev_xmit_skb(struct sk_buff * skb,struct Qdisc * q,struct net_device * dev,struct netdev_queue * txq)2654 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2655 				 struct net_device *dev,
2656 				 struct netdev_queue *txq)
2657 {
2658 	spinlock_t *root_lock = qdisc_lock(q);
2659 	bool contended;
2660 	int rc;
2661 
2662 	qdisc_pkt_len_init(skb);
2663 	qdisc_calculate_pkt_len(skb, q);
2664 	/*
2665 	 * Heuristic to force contended enqueues to serialize on a
2666 	 * separate lock before trying to get qdisc main lock.
2667 	 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2668 	 * and dequeue packets faster.
2669 	 */
2670 	contended = qdisc_is_running(q);
2671 	if (unlikely(contended))
2672 		spin_lock(&q->busylock);
2673 
2674 	spin_lock(root_lock);
2675 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2676 		kfree_skb(skb);
2677 		rc = NET_XMIT_DROP;
2678 	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2679 		   qdisc_run_begin(q)) {
2680 		/*
2681 		 * This is a work-conserving queue; there are no old skbs
2682 		 * waiting to be sent out; and the qdisc is not running -
2683 		 * xmit the skb directly.
2684 		 */
2685 		if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2686 			skb_dst_force(skb);
2687 
2688 		qdisc_bstats_update(q, skb);
2689 
2690 		if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2691 			if (unlikely(contended)) {
2692 				spin_unlock(&q->busylock);
2693 				contended = false;
2694 			}
2695 			__qdisc_run(q);
2696 		} else
2697 			qdisc_run_end(q);
2698 
2699 		rc = NET_XMIT_SUCCESS;
2700 	} else {
2701 		skb_dst_force(skb);
2702 		rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2703 		if (qdisc_run_begin(q)) {
2704 			if (unlikely(contended)) {
2705 				spin_unlock(&q->busylock);
2706 				contended = false;
2707 			}
2708 			__qdisc_run(q);
2709 		}
2710 	}
2711 	spin_unlock(root_lock);
2712 	if (unlikely(contended))
2713 		spin_unlock(&q->busylock);
2714 	return rc;
2715 }
2716 
2717 #if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
skb_update_prio(struct sk_buff * skb)2718 static void skb_update_prio(struct sk_buff *skb)
2719 {
2720 	struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
2721 
2722 	if (!skb->priority && skb->sk && map) {
2723 		unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2724 
2725 		if (prioidx < map->priomap_len)
2726 			skb->priority = map->priomap[prioidx];
2727 	}
2728 }
2729 #else
2730 #define skb_update_prio(skb)
2731 #endif
2732 
2733 static DEFINE_PER_CPU(int, xmit_recursion);
2734 #define RECURSION_LIMIT 10
2735 
2736 /**
2737  *	dev_loopback_xmit - loop back @skb
2738  *	@skb: buffer to transmit
2739  */
dev_loopback_xmit(struct sk_buff * skb)2740 int dev_loopback_xmit(struct sk_buff *skb)
2741 {
2742 	skb_reset_mac_header(skb);
2743 	__skb_pull(skb, skb_network_offset(skb));
2744 	skb->pkt_type = PACKET_LOOPBACK;
2745 	skb->ip_summed = CHECKSUM_UNNECESSARY;
2746 	WARN_ON(!skb_dst(skb));
2747 	skb_dst_force(skb);
2748 	netif_rx_ni(skb);
2749 	return 0;
2750 }
2751 EXPORT_SYMBOL(dev_loopback_xmit);
2752 
2753 /**
2754  *	dev_queue_xmit - transmit a buffer
2755  *	@skb: buffer to transmit
2756  *
2757  *	Queue a buffer for transmission to a network device. The caller must
2758  *	have set the device and priority and built the buffer before calling
2759  *	this function. The function can be called from an interrupt.
2760  *
2761  *	A negative errno code is returned on a failure. A success does not
2762  *	guarantee the frame will be transmitted as it may be dropped due
2763  *	to congestion or traffic shaping.
2764  *
2765  * -----------------------------------------------------------------------------------
2766  *      I notice this method can also return errors from the queue disciplines,
2767  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
2768  *      be positive.
2769  *
2770  *      Regardless of the return value, the skb is consumed, so it is currently
2771  *      difficult to retry a send to this method.  (You can bump the ref count
2772  *      before sending to hold a reference for retry if you are careful.)
2773  *
2774  *      When calling this method, interrupts MUST be enabled.  This is because
2775  *      the BH enable code must have IRQs enabled so that it will not deadlock.
2776  *          --BLG
2777  */
dev_queue_xmit(struct sk_buff * skb)2778 int dev_queue_xmit(struct sk_buff *skb)
2779 {
2780 	struct net_device *dev = skb->dev;
2781 	struct netdev_queue *txq;
2782 	struct Qdisc *q;
2783 	int rc = -ENOMEM;
2784 
2785 	skb_reset_mac_header(skb);
2786 
2787 	/* Disable soft irqs for various locks below. Also
2788 	 * stops preemption for RCU.
2789 	 */
2790 	rcu_read_lock_bh();
2791 
2792 	skb_update_prio(skb);
2793 
2794 	txq = netdev_pick_tx(dev, skb);
2795 	q = rcu_dereference_bh(txq->qdisc);
2796 
2797 #ifdef CONFIG_NET_CLS_ACT
2798 	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2799 #endif
2800 	trace_net_dev_queue(skb);
2801 	if (q->enqueue) {
2802 		rc = __dev_xmit_skb(skb, q, dev, txq);
2803 		goto out;
2804 	}
2805 
2806 	/* The device has no queue. Common case for software devices:
2807 	   loopback, all the sorts of tunnels...
2808 
2809 	   Really, it is unlikely that netif_tx_lock protection is necessary
2810 	   here.  (f.e. loopback and IP tunnels are clean ignoring statistics
2811 	   counters.)
2812 	   However, it is possible, that they rely on protection
2813 	   made by us here.
2814 
2815 	   Check this and shot the lock. It is not prone from deadlocks.
2816 	   Either shot noqueue qdisc, it is even simpler 8)
2817 	 */
2818 	if (dev->flags & IFF_UP) {
2819 		int cpu = smp_processor_id(); /* ok because BHs are off */
2820 
2821 		if (txq->xmit_lock_owner != cpu) {
2822 
2823 			if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2824 				goto recursion_alert;
2825 
2826 			HARD_TX_LOCK(dev, txq, cpu);
2827 
2828 			if (!netif_xmit_stopped(txq)) {
2829 				__this_cpu_inc(xmit_recursion);
2830 				rc = dev_hard_start_xmit(skb, dev, txq);
2831 				__this_cpu_dec(xmit_recursion);
2832 				if (dev_xmit_complete(rc)) {
2833 					HARD_TX_UNLOCK(dev, txq);
2834 					goto out;
2835 				}
2836 			}
2837 			HARD_TX_UNLOCK(dev, txq);
2838 			net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2839 					     dev->name);
2840 		} else {
2841 			/* Recursion is detected! It is possible,
2842 			 * unfortunately
2843 			 */
2844 recursion_alert:
2845 			net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2846 					     dev->name);
2847 		}
2848 	}
2849 
2850 	rc = -ENETDOWN;
2851 	rcu_read_unlock_bh();
2852 
2853 	kfree_skb(skb);
2854 	return rc;
2855 out:
2856 	rcu_read_unlock_bh();
2857 	return rc;
2858 }
2859 EXPORT_SYMBOL(dev_queue_xmit);
2860 
2861 
2862 /*=======================================================================
2863 			Receiver routines
2864   =======================================================================*/
2865 
2866 int netdev_max_backlog __read_mostly = 1000;
2867 EXPORT_SYMBOL(netdev_max_backlog);
2868 
2869 int netdev_tstamp_prequeue __read_mostly = 1;
2870 int netdev_budget __read_mostly = 300;
2871 int weight_p __read_mostly = 64;            /* old backlog weight */
2872 
2873 /* Called with irq disabled */
____napi_schedule(struct softnet_data * sd,struct napi_struct * napi)2874 static inline void ____napi_schedule(struct softnet_data *sd,
2875 				     struct napi_struct *napi)
2876 {
2877 	list_add_tail(&napi->poll_list, &sd->poll_list);
2878 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
2879 }
2880 
2881 #ifdef CONFIG_RPS
2882 
2883 /* One global table that all flow-based protocols share. */
2884 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2885 EXPORT_SYMBOL(rps_sock_flow_table);
2886 
2887 struct static_key rps_needed __read_mostly;
2888 
2889 static struct rps_dev_flow *
set_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow * rflow,u16 next_cpu)2890 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2891 	    struct rps_dev_flow *rflow, u16 next_cpu)
2892 {
2893 	if (next_cpu != RPS_NO_CPU) {
2894 #ifdef CONFIG_RFS_ACCEL
2895 		struct netdev_rx_queue *rxqueue;
2896 		struct rps_dev_flow_table *flow_table;
2897 		struct rps_dev_flow *old_rflow;
2898 		u32 flow_id;
2899 		u16 rxq_index;
2900 		int rc;
2901 
2902 		/* Should we steer this flow to a different hardware queue? */
2903 		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2904 		    !(dev->features & NETIF_F_NTUPLE))
2905 			goto out;
2906 		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2907 		if (rxq_index == skb_get_rx_queue(skb))
2908 			goto out;
2909 
2910 		rxqueue = dev->_rx + rxq_index;
2911 		flow_table = rcu_dereference(rxqueue->rps_flow_table);
2912 		if (!flow_table)
2913 			goto out;
2914 		flow_id = skb->rxhash & flow_table->mask;
2915 		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2916 							rxq_index, flow_id);
2917 		if (rc < 0)
2918 			goto out;
2919 		old_rflow = rflow;
2920 		rflow = &flow_table->flows[flow_id];
2921 		rflow->filter = rc;
2922 		if (old_rflow->filter == rflow->filter)
2923 			old_rflow->filter = RPS_NO_FILTER;
2924 	out:
2925 #endif
2926 		rflow->last_qtail =
2927 			per_cpu(softnet_data, next_cpu).input_queue_head;
2928 	}
2929 
2930 	rflow->cpu = next_cpu;
2931 	return rflow;
2932 }
2933 
2934 /*
2935  * get_rps_cpu is called from netif_receive_skb and returns the target
2936  * CPU from the RPS map of the receiving queue for a given skb.
2937  * rcu_read_lock must be held on entry.
2938  */
get_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow ** rflowp)2939 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2940 		       struct rps_dev_flow **rflowp)
2941 {
2942 	struct netdev_rx_queue *rxqueue;
2943 	struct rps_map *map;
2944 	struct rps_dev_flow_table *flow_table;
2945 	struct rps_sock_flow_table *sock_flow_table;
2946 	int cpu = -1;
2947 	u16 tcpu;
2948 
2949 	if (skb_rx_queue_recorded(skb)) {
2950 		u16 index = skb_get_rx_queue(skb);
2951 		if (unlikely(index >= dev->real_num_rx_queues)) {
2952 			WARN_ONCE(dev->real_num_rx_queues > 1,
2953 				  "%s received packet on queue %u, but number "
2954 				  "of RX queues is %u\n",
2955 				  dev->name, index, dev->real_num_rx_queues);
2956 			goto done;
2957 		}
2958 		rxqueue = dev->_rx + index;
2959 	} else
2960 		rxqueue = dev->_rx;
2961 
2962 	map = rcu_dereference(rxqueue->rps_map);
2963 	if (map) {
2964 		if (map->len == 1 &&
2965 		    !rcu_access_pointer(rxqueue->rps_flow_table)) {
2966 			tcpu = map->cpus[0];
2967 			if (cpu_online(tcpu))
2968 				cpu = tcpu;
2969 			goto done;
2970 		}
2971 	} else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
2972 		goto done;
2973 	}
2974 
2975 	skb_reset_network_header(skb);
2976 	if (!skb_get_rxhash(skb))
2977 		goto done;
2978 
2979 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
2980 	sock_flow_table = rcu_dereference(rps_sock_flow_table);
2981 	if (flow_table && sock_flow_table) {
2982 		u16 next_cpu;
2983 		struct rps_dev_flow *rflow;
2984 
2985 		rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2986 		tcpu = rflow->cpu;
2987 
2988 		next_cpu = sock_flow_table->ents[skb->rxhash &
2989 		    sock_flow_table->mask];
2990 
2991 		/*
2992 		 * If the desired CPU (where last recvmsg was done) is
2993 		 * different from current CPU (one in the rx-queue flow
2994 		 * table entry), switch if one of the following holds:
2995 		 *   - Current CPU is unset (equal to RPS_NO_CPU).
2996 		 *   - Current CPU is offline.
2997 		 *   - The current CPU's queue tail has advanced beyond the
2998 		 *     last packet that was enqueued using this table entry.
2999 		 *     This guarantees that all previous packets for the flow
3000 		 *     have been dequeued, thus preserving in order delivery.
3001 		 */
3002 		if (unlikely(tcpu != next_cpu) &&
3003 		    (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
3004 		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3005 		      rflow->last_qtail)) >= 0)) {
3006 			tcpu = next_cpu;
3007 			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3008 		}
3009 
3010 		if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
3011 			*rflowp = rflow;
3012 			cpu = tcpu;
3013 			goto done;
3014 		}
3015 	}
3016 
3017 	if (map) {
3018 		tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
3019 
3020 		if (cpu_online(tcpu)) {
3021 			cpu = tcpu;
3022 			goto done;
3023 		}
3024 	}
3025 
3026 done:
3027 	return cpu;
3028 }
3029 
3030 #ifdef CONFIG_RFS_ACCEL
3031 
3032 /**
3033  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3034  * @dev: Device on which the filter was set
3035  * @rxq_index: RX queue index
3036  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3037  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3038  *
3039  * Drivers that implement ndo_rx_flow_steer() should periodically call
3040  * this function for each installed filter and remove the filters for
3041  * which it returns %true.
3042  */
rps_may_expire_flow(struct net_device * dev,u16 rxq_index,u32 flow_id,u16 filter_id)3043 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3044 			 u32 flow_id, u16 filter_id)
3045 {
3046 	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3047 	struct rps_dev_flow_table *flow_table;
3048 	struct rps_dev_flow *rflow;
3049 	bool expire = true;
3050 	int cpu;
3051 
3052 	rcu_read_lock();
3053 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3054 	if (flow_table && flow_id <= flow_table->mask) {
3055 		rflow = &flow_table->flows[flow_id];
3056 		cpu = ACCESS_ONCE(rflow->cpu);
3057 		if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
3058 		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3059 			   rflow->last_qtail) <
3060 		     (int)(10 * flow_table->mask)))
3061 			expire = false;
3062 	}
3063 	rcu_read_unlock();
3064 	return expire;
3065 }
3066 EXPORT_SYMBOL(rps_may_expire_flow);
3067 
3068 #endif /* CONFIG_RFS_ACCEL */
3069 
3070 /* Called from hardirq (IPI) context */
rps_trigger_softirq(void * data)3071 static void rps_trigger_softirq(void *data)
3072 {
3073 	struct softnet_data *sd = data;
3074 
3075 	____napi_schedule(sd, &sd->backlog);
3076 	sd->received_rps++;
3077 }
3078 
3079 #endif /* CONFIG_RPS */
3080 
3081 /*
3082  * Check if this softnet_data structure is another cpu one
3083  * If yes, queue it to our IPI list and return 1
3084  * If no, return 0
3085  */
rps_ipi_queued(struct softnet_data * sd)3086 static int rps_ipi_queued(struct softnet_data *sd)
3087 {
3088 #ifdef CONFIG_RPS
3089 	struct softnet_data *mysd = &__get_cpu_var(softnet_data);
3090 
3091 	if (sd != mysd) {
3092 		sd->rps_ipi_next = mysd->rps_ipi_list;
3093 		mysd->rps_ipi_list = sd;
3094 
3095 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3096 		return 1;
3097 	}
3098 #endif /* CONFIG_RPS */
3099 	return 0;
3100 }
3101 
3102 /*
3103  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3104  * queue (may be a remote CPU queue).
3105  */
enqueue_to_backlog(struct sk_buff * skb,int cpu,unsigned int * qtail)3106 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3107 			      unsigned int *qtail)
3108 {
3109 	struct softnet_data *sd;
3110 	unsigned long flags;
3111 
3112 	sd = &per_cpu(softnet_data, cpu);
3113 
3114 	local_irq_save(flags);
3115 
3116 	rps_lock(sd);
3117 	if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
3118 		if (skb_queue_len(&sd->input_pkt_queue)) {
3119 enqueue:
3120 			__skb_queue_tail(&sd->input_pkt_queue, skb);
3121 			input_queue_tail_incr_save(sd, qtail);
3122 			rps_unlock(sd);
3123 			local_irq_restore(flags);
3124 			return NET_RX_SUCCESS;
3125 		}
3126 
3127 		/* Schedule NAPI for backlog device
3128 		 * We can use non atomic operation since we own the queue lock
3129 		 */
3130 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3131 			if (!rps_ipi_queued(sd))
3132 				____napi_schedule(sd, &sd->backlog);
3133 		}
3134 		goto enqueue;
3135 	}
3136 
3137 	sd->dropped++;
3138 	rps_unlock(sd);
3139 
3140 	local_irq_restore(flags);
3141 
3142 	atomic_long_inc(&skb->dev->rx_dropped);
3143 	kfree_skb(skb);
3144 	return NET_RX_DROP;
3145 }
3146 
3147 /**
3148  *	netif_rx	-	post buffer to the network code
3149  *	@skb: buffer to post
3150  *
3151  *	This function receives a packet from a device driver and queues it for
3152  *	the upper (protocol) levels to process.  It always succeeds. The buffer
3153  *	may be dropped during processing for congestion control or by the
3154  *	protocol layers.
3155  *
3156  *	return values:
3157  *	NET_RX_SUCCESS	(no congestion)
3158  *	NET_RX_DROP     (packet was dropped)
3159  *
3160  */
3161 
netif_rx(struct sk_buff * skb)3162 int netif_rx(struct sk_buff *skb)
3163 {
3164 	int ret;
3165 
3166 	/* if netpoll wants it, pretend we never saw it */
3167 	if (netpoll_rx(skb))
3168 		return NET_RX_DROP;
3169 
3170 	net_timestamp_check(netdev_tstamp_prequeue, skb);
3171 
3172 	trace_netif_rx(skb);
3173 #ifdef CONFIG_RPS
3174 	if (static_key_false(&rps_needed)) {
3175 		struct rps_dev_flow voidflow, *rflow = &voidflow;
3176 		int cpu;
3177 
3178 		preempt_disable();
3179 		rcu_read_lock();
3180 
3181 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
3182 		if (cpu < 0)
3183 			cpu = smp_processor_id();
3184 
3185 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3186 
3187 		rcu_read_unlock();
3188 		preempt_enable();
3189 	} else
3190 #endif
3191 	{
3192 		unsigned int qtail;
3193 		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3194 		put_cpu();
3195 	}
3196 	return ret;
3197 }
3198 EXPORT_SYMBOL(netif_rx);
3199 
netif_rx_ni(struct sk_buff * skb)3200 int netif_rx_ni(struct sk_buff *skb)
3201 {
3202 	int err;
3203 
3204 	preempt_disable();
3205 	err = netif_rx(skb);
3206 	if (local_softirq_pending())
3207 		do_softirq();
3208 	preempt_enable();
3209 
3210 	return err;
3211 }
3212 EXPORT_SYMBOL(netif_rx_ni);
3213 
net_tx_action(struct softirq_action * h)3214 static void net_tx_action(struct softirq_action *h)
3215 {
3216 	struct softnet_data *sd = &__get_cpu_var(softnet_data);
3217 
3218 	if (sd->completion_queue) {
3219 		struct sk_buff *clist;
3220 
3221 		local_irq_disable();
3222 		clist = sd->completion_queue;
3223 		sd->completion_queue = NULL;
3224 		local_irq_enable();
3225 
3226 		while (clist) {
3227 			struct sk_buff *skb = clist;
3228 			clist = clist->next;
3229 
3230 			WARN_ON(atomic_read(&skb->users));
3231 			trace_kfree_skb(skb, net_tx_action);
3232 			__kfree_skb(skb);
3233 		}
3234 	}
3235 
3236 	if (sd->output_queue) {
3237 		struct Qdisc *head;
3238 
3239 		local_irq_disable();
3240 		head = sd->output_queue;
3241 		sd->output_queue = NULL;
3242 		sd->output_queue_tailp = &sd->output_queue;
3243 		local_irq_enable();
3244 
3245 		while (head) {
3246 			struct Qdisc *q = head;
3247 			spinlock_t *root_lock;
3248 
3249 			head = head->next_sched;
3250 
3251 			root_lock = qdisc_lock(q);
3252 			if (spin_trylock(root_lock)) {
3253 				smp_mb__before_clear_bit();
3254 				clear_bit(__QDISC_STATE_SCHED,
3255 					  &q->state);
3256 				qdisc_run(q);
3257 				spin_unlock(root_lock);
3258 			} else {
3259 				if (!test_bit(__QDISC_STATE_DEACTIVATED,
3260 					      &q->state)) {
3261 					__netif_reschedule(q);
3262 				} else {
3263 					smp_mb__before_clear_bit();
3264 					clear_bit(__QDISC_STATE_SCHED,
3265 						  &q->state);
3266 				}
3267 			}
3268 		}
3269 	}
3270 }
3271 
3272 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3273     (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3274 /* This hook is defined here for ATM LANE */
3275 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3276 			     unsigned char *addr) __read_mostly;
3277 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3278 #endif
3279 
3280 #ifdef CONFIG_NET_CLS_ACT
3281 /* TODO: Maybe we should just force sch_ingress to be compiled in
3282  * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3283  * a compare and 2 stores extra right now if we dont have it on
3284  * but have CONFIG_NET_CLS_ACT
3285  * NOTE: This doesn't stop any functionality; if you dont have
3286  * the ingress scheduler, you just can't add policies on ingress.
3287  *
3288  */
ing_filter(struct sk_buff * skb,struct netdev_queue * rxq)3289 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
3290 {
3291 	struct net_device *dev = skb->dev;
3292 	u32 ttl = G_TC_RTTL(skb->tc_verd);
3293 	int result = TC_ACT_OK;
3294 	struct Qdisc *q;
3295 
3296 	if (unlikely(MAX_RED_LOOP < ttl++)) {
3297 		net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3298 				     skb->skb_iif, dev->ifindex);
3299 		return TC_ACT_SHOT;
3300 	}
3301 
3302 	skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3303 	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3304 
3305 	q = rxq->qdisc;
3306 	if (q != &noop_qdisc) {
3307 		spin_lock(qdisc_lock(q));
3308 		if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3309 			result = qdisc_enqueue_root(skb, q);
3310 		spin_unlock(qdisc_lock(q));
3311 	}
3312 
3313 	return result;
3314 }
3315 
handle_ing(struct sk_buff * skb,struct packet_type ** pt_prev,int * ret,struct net_device * orig_dev)3316 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3317 					 struct packet_type **pt_prev,
3318 					 int *ret, struct net_device *orig_dev)
3319 {
3320 	struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3321 
3322 	if (!rxq || rxq->qdisc == &noop_qdisc)
3323 		goto out;
3324 
3325 	if (*pt_prev) {
3326 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
3327 		*pt_prev = NULL;
3328 	}
3329 
3330 	switch (ing_filter(skb, rxq)) {
3331 	case TC_ACT_SHOT:
3332 	case TC_ACT_STOLEN:
3333 		kfree_skb(skb);
3334 		return NULL;
3335 	}
3336 
3337 out:
3338 	skb->tc_verd = 0;
3339 	return skb;
3340 }
3341 #endif
3342 
3343 /**
3344  *	netdev_rx_handler_register - register receive handler
3345  *	@dev: device to register a handler for
3346  *	@rx_handler: receive handler to register
3347  *	@rx_handler_data: data pointer that is used by rx handler
3348  *
3349  *	Register a receive hander for a device. This handler will then be
3350  *	called from __netif_receive_skb. A negative errno code is returned
3351  *	on a failure.
3352  *
3353  *	The caller must hold the rtnl_mutex.
3354  *
3355  *	For a general description of rx_handler, see enum rx_handler_result.
3356  */
netdev_rx_handler_register(struct net_device * dev,rx_handler_func_t * rx_handler,void * rx_handler_data)3357 int netdev_rx_handler_register(struct net_device *dev,
3358 			       rx_handler_func_t *rx_handler,
3359 			       void *rx_handler_data)
3360 {
3361 	ASSERT_RTNL();
3362 
3363 	if (dev->rx_handler)
3364 		return -EBUSY;
3365 
3366 	/* Note: rx_handler_data must be set before rx_handler */
3367 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3368 	rcu_assign_pointer(dev->rx_handler, rx_handler);
3369 
3370 	return 0;
3371 }
3372 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3373 
3374 /**
3375  *	netdev_rx_handler_unregister - unregister receive handler
3376  *	@dev: device to unregister a handler from
3377  *
3378  *	Unregister a receive handler from a device.
3379  *
3380  *	The caller must hold the rtnl_mutex.
3381  */
netdev_rx_handler_unregister(struct net_device * dev)3382 void netdev_rx_handler_unregister(struct net_device *dev)
3383 {
3384 
3385 	ASSERT_RTNL();
3386 	RCU_INIT_POINTER(dev->rx_handler, NULL);
3387 	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3388 	 * section has a guarantee to see a non NULL rx_handler_data
3389 	 * as well.
3390 	 */
3391 	synchronize_net();
3392 	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3393 }
3394 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3395 
3396 /*
3397  * Limit the use of PFMEMALLOC reserves to those protocols that implement
3398  * the special handling of PFMEMALLOC skbs.
3399  */
skb_pfmemalloc_protocol(struct sk_buff * skb)3400 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3401 {
3402 	switch (skb->protocol) {
3403 	case __constant_htons(ETH_P_ARP):
3404 	case __constant_htons(ETH_P_IP):
3405 	case __constant_htons(ETH_P_IPV6):
3406 	case __constant_htons(ETH_P_8021Q):
3407 	case __constant_htons(ETH_P_8021AD):
3408 		return true;
3409 	default:
3410 		return false;
3411 	}
3412 }
3413 
__netif_receive_skb_core(struct sk_buff * skb,bool pfmemalloc)3414 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
3415 {
3416 	struct packet_type *ptype, *pt_prev;
3417 	rx_handler_func_t *rx_handler;
3418 	struct net_device *orig_dev;
3419 	struct net_device *null_or_dev;
3420 	bool deliver_exact = false;
3421 	int ret = NET_RX_DROP;
3422 	__be16 type;
3423 
3424 	net_timestamp_check(!netdev_tstamp_prequeue, skb);
3425 
3426 	trace_netif_receive_skb(skb);
3427 
3428 	/* if we've gotten here through NAPI, check netpoll */
3429 	if (netpoll_receive_skb(skb))
3430 		goto out;
3431 
3432 	orig_dev = skb->dev;
3433 
3434 	skb_reset_network_header(skb);
3435 	if (!skb_transport_header_was_set(skb))
3436 		skb_reset_transport_header(skb);
3437 	skb_reset_mac_len(skb);
3438 
3439 	pt_prev = NULL;
3440 
3441 	rcu_read_lock();
3442 
3443 another_round:
3444 	skb->skb_iif = skb->dev->ifindex;
3445 
3446 	__this_cpu_inc(softnet_data.processed);
3447 
3448 	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3449 	    skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
3450 		skb = vlan_untag(skb);
3451 		if (unlikely(!skb))
3452 			goto unlock;
3453 	}
3454 
3455 #ifdef CONFIG_NET_CLS_ACT
3456 	if (skb->tc_verd & TC_NCLS) {
3457 		skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3458 		goto ncls;
3459 	}
3460 #endif
3461 
3462 	if (pfmemalloc)
3463 		goto skip_taps;
3464 
3465 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
3466 		if (!ptype->dev || ptype->dev == skb->dev) {
3467 			if (pt_prev)
3468 				ret = deliver_skb(skb, pt_prev, orig_dev);
3469 			pt_prev = ptype;
3470 		}
3471 	}
3472 
3473 skip_taps:
3474 #ifdef CONFIG_NET_CLS_ACT
3475 	skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3476 	if (!skb)
3477 		goto unlock;
3478 ncls:
3479 #endif
3480 
3481 	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
3482 		goto drop;
3483 
3484 	if (vlan_tx_tag_present(skb)) {
3485 		if (pt_prev) {
3486 			ret = deliver_skb(skb, pt_prev, orig_dev);
3487 			pt_prev = NULL;
3488 		}
3489 		if (vlan_do_receive(&skb))
3490 			goto another_round;
3491 		else if (unlikely(!skb))
3492 			goto unlock;
3493 	}
3494 
3495 	rx_handler = rcu_dereference(skb->dev->rx_handler);
3496 	if (rx_handler) {
3497 		if (pt_prev) {
3498 			ret = deliver_skb(skb, pt_prev, orig_dev);
3499 			pt_prev = NULL;
3500 		}
3501 		switch (rx_handler(&skb)) {
3502 		case RX_HANDLER_CONSUMED:
3503 			ret = NET_RX_SUCCESS;
3504 			goto unlock;
3505 		case RX_HANDLER_ANOTHER:
3506 			goto another_round;
3507 		case RX_HANDLER_EXACT:
3508 			deliver_exact = true;
3509 		case RX_HANDLER_PASS:
3510 			break;
3511 		default:
3512 			BUG();
3513 		}
3514 	}
3515 
3516 	if (vlan_tx_nonzero_tag_present(skb))
3517 		skb->pkt_type = PACKET_OTHERHOST;
3518 
3519 	/* deliver only exact match when indicated */
3520 	null_or_dev = deliver_exact ? skb->dev : NULL;
3521 
3522 	type = skb->protocol;
3523 	list_for_each_entry_rcu(ptype,
3524 			&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3525 		if (ptype->type == type &&
3526 		    (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3527 		     ptype->dev == orig_dev)) {
3528 			if (pt_prev)
3529 				ret = deliver_skb(skb, pt_prev, orig_dev);
3530 			pt_prev = ptype;
3531 		}
3532 	}
3533 
3534 	if (pt_prev) {
3535 		if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3536 			goto drop;
3537 		else
3538 			ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3539 	} else {
3540 drop:
3541 		atomic_long_inc(&skb->dev->rx_dropped);
3542 		kfree_skb(skb);
3543 		/* Jamal, now you will not able to escape explaining
3544 		 * me how you were going to use this. :-)
3545 		 */
3546 		ret = NET_RX_DROP;
3547 	}
3548 
3549 unlock:
3550 	rcu_read_unlock();
3551 out:
3552 	return ret;
3553 }
3554 
__netif_receive_skb(struct sk_buff * skb)3555 static int __netif_receive_skb(struct sk_buff *skb)
3556 {
3557 	int ret;
3558 
3559 	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3560 		unsigned long pflags = current->flags;
3561 
3562 		/*
3563 		 * PFMEMALLOC skbs are special, they should
3564 		 * - be delivered to SOCK_MEMALLOC sockets only
3565 		 * - stay away from userspace
3566 		 * - have bounded memory usage
3567 		 *
3568 		 * Use PF_MEMALLOC as this saves us from propagating the allocation
3569 		 * context down to all allocation sites.
3570 		 */
3571 		current->flags |= PF_MEMALLOC;
3572 		ret = __netif_receive_skb_core(skb, true);
3573 		tsk_restore_flags(current, pflags, PF_MEMALLOC);
3574 	} else
3575 		ret = __netif_receive_skb_core(skb, false);
3576 
3577 	return ret;
3578 }
3579 
3580 /**
3581  *	netif_receive_skb - process receive buffer from network
3582  *	@skb: buffer to process
3583  *
3584  *	netif_receive_skb() is the main receive data processing function.
3585  *	It always succeeds. The buffer may be dropped during processing
3586  *	for congestion control or by the protocol layers.
3587  *
3588  *	This function may only be called from softirq context and interrupts
3589  *	should be enabled.
3590  *
3591  *	Return values (usually ignored):
3592  *	NET_RX_SUCCESS: no congestion
3593  *	NET_RX_DROP: packet was dropped
3594  */
netif_receive_skb(struct sk_buff * skb)3595 int netif_receive_skb(struct sk_buff *skb)
3596 {
3597 	net_timestamp_check(netdev_tstamp_prequeue, skb);
3598 
3599 	if (skb_defer_rx_timestamp(skb))
3600 		return NET_RX_SUCCESS;
3601 
3602 #ifdef CONFIG_RPS
3603 	if (static_key_false(&rps_needed)) {
3604 		struct rps_dev_flow voidflow, *rflow = &voidflow;
3605 		int cpu, ret;
3606 
3607 		rcu_read_lock();
3608 
3609 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
3610 
3611 		if (cpu >= 0) {
3612 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3613 			rcu_read_unlock();
3614 			return ret;
3615 		}
3616 		rcu_read_unlock();
3617 	}
3618 #endif
3619 	return __netif_receive_skb(skb);
3620 }
3621 EXPORT_SYMBOL(netif_receive_skb);
3622 
3623 /* Network device is going away, flush any packets still pending
3624  * Called with irqs disabled.
3625  */
flush_backlog(void * arg)3626 static void flush_backlog(void *arg)
3627 {
3628 	struct net_device *dev = arg;
3629 	struct softnet_data *sd = &__get_cpu_var(softnet_data);
3630 	struct sk_buff *skb, *tmp;
3631 
3632 	rps_lock(sd);
3633 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3634 		if (skb->dev == dev) {
3635 			__skb_unlink(skb, &sd->input_pkt_queue);
3636 			kfree_skb(skb);
3637 			input_queue_head_incr(sd);
3638 		}
3639 	}
3640 	rps_unlock(sd);
3641 
3642 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3643 		if (skb->dev == dev) {
3644 			__skb_unlink(skb, &sd->process_queue);
3645 			kfree_skb(skb);
3646 			input_queue_head_incr(sd);
3647 		}
3648 	}
3649 }
3650 
napi_gro_complete(struct sk_buff * skb)3651 static int napi_gro_complete(struct sk_buff *skb)
3652 {
3653 	struct packet_offload *ptype;
3654 	__be16 type = skb->protocol;
3655 	struct list_head *head = &offload_base;
3656 	int err = -ENOENT;
3657 
3658 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
3659 
3660 	if (NAPI_GRO_CB(skb)->count == 1) {
3661 		skb_shinfo(skb)->gso_size = 0;
3662 		goto out;
3663 	}
3664 
3665 	rcu_read_lock();
3666 	list_for_each_entry_rcu(ptype, head, list) {
3667 		if (ptype->type != type || !ptype->callbacks.gro_complete)
3668 			continue;
3669 
3670 		err = ptype->callbacks.gro_complete(skb);
3671 		break;
3672 	}
3673 	rcu_read_unlock();
3674 
3675 	if (err) {
3676 		WARN_ON(&ptype->list == head);
3677 		kfree_skb(skb);
3678 		return NET_RX_SUCCESS;
3679 	}
3680 
3681 out:
3682 	return netif_receive_skb(skb);
3683 }
3684 
3685 /* napi->gro_list contains packets ordered by age.
3686  * youngest packets at the head of it.
3687  * Complete skbs in reverse order to reduce latencies.
3688  */
napi_gro_flush(struct napi_struct * napi,bool flush_old)3689 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
3690 {
3691 	struct sk_buff *skb, *prev = NULL;
3692 
3693 	/* scan list and build reverse chain */
3694 	for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3695 		skb->prev = prev;
3696 		prev = skb;
3697 	}
3698 
3699 	for (skb = prev; skb; skb = prev) {
3700 		skb->next = NULL;
3701 
3702 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3703 			return;
3704 
3705 		prev = skb->prev;
3706 		napi_gro_complete(skb);
3707 		napi->gro_count--;
3708 	}
3709 
3710 	napi->gro_list = NULL;
3711 }
3712 EXPORT_SYMBOL(napi_gro_flush);
3713 
gro_list_prepare(struct napi_struct * napi,struct sk_buff * skb)3714 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
3715 {
3716 	struct sk_buff *p;
3717 	unsigned int maclen = skb->dev->hard_header_len;
3718 
3719 	for (p = napi->gro_list; p; p = p->next) {
3720 		unsigned long diffs;
3721 
3722 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3723 		diffs |= p->vlan_tci ^ skb->vlan_tci;
3724 		if (maclen == ETH_HLEN)
3725 			diffs |= compare_ether_header(skb_mac_header(p),
3726 						      skb_gro_mac_header(skb));
3727 		else if (!diffs)
3728 			diffs = memcmp(skb_mac_header(p),
3729 				       skb_gro_mac_header(skb),
3730 				       maclen);
3731 		NAPI_GRO_CB(p)->same_flow = !diffs;
3732 		NAPI_GRO_CB(p)->flush = 0;
3733 	}
3734 }
3735 
dev_gro_receive(struct napi_struct * napi,struct sk_buff * skb)3736 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3737 {
3738 	struct sk_buff **pp = NULL;
3739 	struct packet_offload *ptype;
3740 	__be16 type = skb->protocol;
3741 	struct list_head *head = &offload_base;
3742 	int same_flow;
3743 	enum gro_result ret;
3744 
3745 	if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3746 		goto normal;
3747 
3748 	if (skb_is_gso(skb) || skb_has_frag_list(skb))
3749 		goto normal;
3750 
3751 	gro_list_prepare(napi, skb);
3752 
3753 	rcu_read_lock();
3754 	list_for_each_entry_rcu(ptype, head, list) {
3755 		if (ptype->type != type || !ptype->callbacks.gro_receive)
3756 			continue;
3757 
3758 		skb_set_network_header(skb, skb_gro_offset(skb));
3759 		skb_reset_mac_len(skb);
3760 		NAPI_GRO_CB(skb)->same_flow = 0;
3761 		NAPI_GRO_CB(skb)->flush = 0;
3762 		NAPI_GRO_CB(skb)->free = 0;
3763 
3764 		pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
3765 		break;
3766 	}
3767 	rcu_read_unlock();
3768 
3769 	if (&ptype->list == head)
3770 		goto normal;
3771 
3772 	same_flow = NAPI_GRO_CB(skb)->same_flow;
3773 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3774 
3775 	if (pp) {
3776 		struct sk_buff *nskb = *pp;
3777 
3778 		*pp = nskb->next;
3779 		nskb->next = NULL;
3780 		napi_gro_complete(nskb);
3781 		napi->gro_count--;
3782 	}
3783 
3784 	if (same_flow)
3785 		goto ok;
3786 
3787 	if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3788 		goto normal;
3789 
3790 	napi->gro_count++;
3791 	NAPI_GRO_CB(skb)->count = 1;
3792 	NAPI_GRO_CB(skb)->age = jiffies;
3793 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3794 	skb->next = napi->gro_list;
3795 	napi->gro_list = skb;
3796 	ret = GRO_HELD;
3797 
3798 pull:
3799 	if (skb_headlen(skb) < skb_gro_offset(skb)) {
3800 		int grow = skb_gro_offset(skb) - skb_headlen(skb);
3801 
3802 		BUG_ON(skb->end - skb->tail < grow);
3803 
3804 		memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3805 
3806 		skb->tail += grow;
3807 		skb->data_len -= grow;
3808 
3809 		skb_shinfo(skb)->frags[0].page_offset += grow;
3810 		skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
3811 
3812 		if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
3813 			skb_frag_unref(skb, 0);
3814 			memmove(skb_shinfo(skb)->frags,
3815 				skb_shinfo(skb)->frags + 1,
3816 				--skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3817 		}
3818 	}
3819 
3820 ok:
3821 	return ret;
3822 
3823 normal:
3824 	ret = GRO_NORMAL;
3825 	goto pull;
3826 }
3827 
3828 
napi_skb_finish(gro_result_t ret,struct sk_buff * skb)3829 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3830 {
3831 	switch (ret) {
3832 	case GRO_NORMAL:
3833 		if (netif_receive_skb(skb))
3834 			ret = GRO_DROP;
3835 		break;
3836 
3837 	case GRO_DROP:
3838 		kfree_skb(skb);
3839 		break;
3840 
3841 	case GRO_MERGED_FREE:
3842 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3843 			kmem_cache_free(skbuff_head_cache, skb);
3844 		else
3845 			__kfree_skb(skb);
3846 		break;
3847 
3848 	case GRO_HELD:
3849 	case GRO_MERGED:
3850 		break;
3851 	}
3852 
3853 	return ret;
3854 }
3855 
skb_gro_reset_offset(struct sk_buff * skb)3856 static void skb_gro_reset_offset(struct sk_buff *skb)
3857 {
3858 	const struct skb_shared_info *pinfo = skb_shinfo(skb);
3859 	const skb_frag_t *frag0 = &pinfo->frags[0];
3860 
3861 	NAPI_GRO_CB(skb)->data_offset = 0;
3862 	NAPI_GRO_CB(skb)->frag0 = NULL;
3863 	NAPI_GRO_CB(skb)->frag0_len = 0;
3864 
3865 	if (skb->mac_header == skb->tail &&
3866 	    pinfo->nr_frags &&
3867 	    !PageHighMem(skb_frag_page(frag0))) {
3868 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3869 		NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
3870 	}
3871 }
3872 
napi_gro_receive(struct napi_struct * napi,struct sk_buff * skb)3873 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3874 {
3875 	skb_gro_reset_offset(skb);
3876 
3877 	return napi_skb_finish(dev_gro_receive(napi, skb), skb);
3878 }
3879 EXPORT_SYMBOL(napi_gro_receive);
3880 
napi_reuse_skb(struct napi_struct * napi,struct sk_buff * skb)3881 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3882 {
3883 	__skb_pull(skb, skb_headlen(skb));
3884 	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
3885 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
3886 	skb->vlan_tci = 0;
3887 	skb->dev = napi->dev;
3888 	skb->skb_iif = 0;
3889 
3890 	napi->skb = skb;
3891 }
3892 
napi_get_frags(struct napi_struct * napi)3893 struct sk_buff *napi_get_frags(struct napi_struct *napi)
3894 {
3895 	struct sk_buff *skb = napi->skb;
3896 
3897 	if (!skb) {
3898 		skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3899 		if (skb)
3900 			napi->skb = skb;
3901 	}
3902 	return skb;
3903 }
3904 EXPORT_SYMBOL(napi_get_frags);
3905 
napi_frags_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)3906 static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3907 			       gro_result_t ret)
3908 {
3909 	switch (ret) {
3910 	case GRO_NORMAL:
3911 	case GRO_HELD:
3912 		skb->protocol = eth_type_trans(skb, skb->dev);
3913 
3914 		if (ret == GRO_HELD)
3915 			skb_gro_pull(skb, -ETH_HLEN);
3916 		else if (netif_receive_skb(skb))
3917 			ret = GRO_DROP;
3918 		break;
3919 
3920 	case GRO_DROP:
3921 	case GRO_MERGED_FREE:
3922 		napi_reuse_skb(napi, skb);
3923 		break;
3924 
3925 	case GRO_MERGED:
3926 		break;
3927 	}
3928 
3929 	return ret;
3930 }
3931 
napi_frags_skb(struct napi_struct * napi)3932 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3933 {
3934 	struct sk_buff *skb = napi->skb;
3935 	struct ethhdr *eth;
3936 	unsigned int hlen;
3937 	unsigned int off;
3938 
3939 	napi->skb = NULL;
3940 
3941 	skb_reset_mac_header(skb);
3942 	skb_gro_reset_offset(skb);
3943 
3944 	off = skb_gro_offset(skb);
3945 	hlen = off + sizeof(*eth);
3946 	eth = skb_gro_header_fast(skb, off);
3947 	if (skb_gro_header_hard(skb, hlen)) {
3948 		eth = skb_gro_header_slow(skb, hlen, off);
3949 		if (unlikely(!eth)) {
3950 			napi_reuse_skb(napi, skb);
3951 			skb = NULL;
3952 			goto out;
3953 		}
3954 	}
3955 
3956 	skb_gro_pull(skb, sizeof(*eth));
3957 
3958 	/*
3959 	 * This works because the only protocols we care about don't require
3960 	 * special handling.  We'll fix it up properly at the end.
3961 	 */
3962 	skb->protocol = eth->h_proto;
3963 
3964 out:
3965 	return skb;
3966 }
3967 
napi_gro_frags(struct napi_struct * napi)3968 gro_result_t napi_gro_frags(struct napi_struct *napi)
3969 {
3970 	struct sk_buff *skb = napi_frags_skb(napi);
3971 
3972 	if (!skb)
3973 		return GRO_DROP;
3974 
3975 	return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
3976 }
3977 EXPORT_SYMBOL(napi_gro_frags);
3978 
3979 /*
3980  * net_rps_action sends any pending IPI's for rps.
3981  * Note: called with local irq disabled, but exits with local irq enabled.
3982  */
net_rps_action_and_irq_enable(struct softnet_data * sd)3983 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3984 {
3985 #ifdef CONFIG_RPS
3986 	struct softnet_data *remsd = sd->rps_ipi_list;
3987 
3988 	if (remsd) {
3989 		sd->rps_ipi_list = NULL;
3990 
3991 		local_irq_enable();
3992 
3993 		/* Send pending IPI's to kick RPS processing on remote cpus. */
3994 		while (remsd) {
3995 			struct softnet_data *next = remsd->rps_ipi_next;
3996 
3997 			if (cpu_online(remsd->cpu))
3998 				__smp_call_function_single(remsd->cpu,
3999 							   &remsd->csd, 0);
4000 			remsd = next;
4001 		}
4002 	} else
4003 #endif
4004 		local_irq_enable();
4005 }
4006 
process_backlog(struct napi_struct * napi,int quota)4007 static int process_backlog(struct napi_struct *napi, int quota)
4008 {
4009 	int work = 0;
4010 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4011 
4012 #ifdef CONFIG_RPS
4013 	/* Check if we have pending ipi, its better to send them now,
4014 	 * not waiting net_rx_action() end.
4015 	 */
4016 	if (sd->rps_ipi_list) {
4017 		local_irq_disable();
4018 		net_rps_action_and_irq_enable(sd);
4019 	}
4020 #endif
4021 	napi->weight = weight_p;
4022 	local_irq_disable();
4023 	while (work < quota) {
4024 		struct sk_buff *skb;
4025 		unsigned int qlen;
4026 
4027 		while ((skb = __skb_dequeue(&sd->process_queue))) {
4028 			local_irq_enable();
4029 			__netif_receive_skb(skb);
4030 			local_irq_disable();
4031 			input_queue_head_incr(sd);
4032 			if (++work >= quota) {
4033 				local_irq_enable();
4034 				return work;
4035 			}
4036 		}
4037 
4038 		rps_lock(sd);
4039 		qlen = skb_queue_len(&sd->input_pkt_queue);
4040 		if (qlen)
4041 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
4042 						   &sd->process_queue);
4043 
4044 		if (qlen < quota - work) {
4045 			/*
4046 			 * Inline a custom version of __napi_complete().
4047 			 * only current cpu owns and manipulates this napi,
4048 			 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
4049 			 * we can use a plain write instead of clear_bit(),
4050 			 * and we dont need an smp_mb() memory barrier.
4051 			 */
4052 			list_del(&napi->poll_list);
4053 			napi->state = 0;
4054 
4055 			quota = work + qlen;
4056 		}
4057 		rps_unlock(sd);
4058 	}
4059 	local_irq_enable();
4060 
4061 	return work;
4062 }
4063 
4064 /**
4065  * __napi_schedule - schedule for receive
4066  * @n: entry to schedule
4067  *
4068  * The entry's receive function will be scheduled to run
4069  */
__napi_schedule(struct napi_struct * n)4070 void __napi_schedule(struct napi_struct *n)
4071 {
4072 	unsigned long flags;
4073 
4074 	local_irq_save(flags);
4075 	____napi_schedule(&__get_cpu_var(softnet_data), n);
4076 	local_irq_restore(flags);
4077 }
4078 EXPORT_SYMBOL(__napi_schedule);
4079 
__napi_complete(struct napi_struct * n)4080 void __napi_complete(struct napi_struct *n)
4081 {
4082 	BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4083 	BUG_ON(n->gro_list);
4084 
4085 	list_del(&n->poll_list);
4086 	smp_mb__before_clear_bit();
4087 	clear_bit(NAPI_STATE_SCHED, &n->state);
4088 }
4089 EXPORT_SYMBOL(__napi_complete);
4090 
napi_complete(struct napi_struct * n)4091 void napi_complete(struct napi_struct *n)
4092 {
4093 	unsigned long flags;
4094 
4095 	/*
4096 	 * don't let napi dequeue from the cpu poll list
4097 	 * just in case its running on a different cpu
4098 	 */
4099 	if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4100 		return;
4101 
4102 	napi_gro_flush(n, false);
4103 	local_irq_save(flags);
4104 	__napi_complete(n);
4105 	local_irq_restore(flags);
4106 }
4107 EXPORT_SYMBOL(napi_complete);
4108 
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)4109 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4110 		    int (*poll)(struct napi_struct *, int), int weight)
4111 {
4112 	INIT_LIST_HEAD(&napi->poll_list);
4113 	napi->gro_count = 0;
4114 	napi->gro_list = NULL;
4115 	napi->skb = NULL;
4116 	napi->poll = poll;
4117 	if (weight > NAPI_POLL_WEIGHT)
4118 		pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4119 			    weight, dev->name);
4120 	napi->weight = weight;
4121 	list_add(&napi->dev_list, &dev->napi_list);
4122 	napi->dev = dev;
4123 #ifdef CONFIG_NETPOLL
4124 	spin_lock_init(&napi->poll_lock);
4125 	napi->poll_owner = -1;
4126 #endif
4127 	set_bit(NAPI_STATE_SCHED, &napi->state);
4128 }
4129 EXPORT_SYMBOL(netif_napi_add);
4130 
netif_napi_del(struct napi_struct * napi)4131 void netif_napi_del(struct napi_struct *napi)
4132 {
4133 	struct sk_buff *skb, *next;
4134 
4135 	list_del_init(&napi->dev_list);
4136 	napi_free_frags(napi);
4137 
4138 	for (skb = napi->gro_list; skb; skb = next) {
4139 		next = skb->next;
4140 		skb->next = NULL;
4141 		kfree_skb(skb);
4142 	}
4143 
4144 	napi->gro_list = NULL;
4145 	napi->gro_count = 0;
4146 }
4147 EXPORT_SYMBOL(netif_napi_del);
4148 
net_rx_action(struct softirq_action * h)4149 static void net_rx_action(struct softirq_action *h)
4150 {
4151 	struct softnet_data *sd = &__get_cpu_var(softnet_data);
4152 	unsigned long time_limit = jiffies + 2;
4153 	int budget = netdev_budget;
4154 	void *have;
4155 
4156 	local_irq_disable();
4157 
4158 	while (!list_empty(&sd->poll_list)) {
4159 		struct napi_struct *n;
4160 		int work, weight;
4161 
4162 		/* If softirq window is exhuasted then punt.
4163 		 * Allow this to run for 2 jiffies since which will allow
4164 		 * an average latency of 1.5/HZ.
4165 		 */
4166 		if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit)))
4167 			goto softnet_break;
4168 
4169 		local_irq_enable();
4170 
4171 		/* Even though interrupts have been re-enabled, this
4172 		 * access is safe because interrupts can only add new
4173 		 * entries to the tail of this list, and only ->poll()
4174 		 * calls can remove this head entry from the list.
4175 		 */
4176 		n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
4177 
4178 		have = netpoll_poll_lock(n);
4179 
4180 		weight = n->weight;
4181 
4182 		/* This NAPI_STATE_SCHED test is for avoiding a race
4183 		 * with netpoll's poll_napi().  Only the entity which
4184 		 * obtains the lock and sees NAPI_STATE_SCHED set will
4185 		 * actually make the ->poll() call.  Therefore we avoid
4186 		 * accidentally calling ->poll() when NAPI is not scheduled.
4187 		 */
4188 		work = 0;
4189 		if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4190 			work = n->poll(n, weight);
4191 			trace_napi_poll(n);
4192 		}
4193 
4194 		WARN_ON_ONCE(work > weight);
4195 
4196 		budget -= work;
4197 
4198 		local_irq_disable();
4199 
4200 		/* Drivers must not modify the NAPI state if they
4201 		 * consume the entire weight.  In such cases this code
4202 		 * still "owns" the NAPI instance and therefore can
4203 		 * move the instance around on the list at-will.
4204 		 */
4205 		if (unlikely(work == weight)) {
4206 			if (unlikely(napi_disable_pending(n))) {
4207 				local_irq_enable();
4208 				napi_complete(n);
4209 				local_irq_disable();
4210 			} else {
4211 				if (n->gro_list) {
4212 					/* flush too old packets
4213 					 * If HZ < 1000, flush all packets.
4214 					 */
4215 					local_irq_enable();
4216 					napi_gro_flush(n, HZ >= 1000);
4217 					local_irq_disable();
4218 				}
4219 				list_move_tail(&n->poll_list, &sd->poll_list);
4220 			}
4221 		}
4222 
4223 		netpoll_poll_unlock(have);
4224 	}
4225 out:
4226 	net_rps_action_and_irq_enable(sd);
4227 
4228 #ifdef CONFIG_NET_DMA
4229 	/*
4230 	 * There may not be any more sk_buffs coming right now, so push
4231 	 * any pending DMA copies to hardware
4232 	 */
4233 	dma_issue_pending_all();
4234 #endif
4235 
4236 	return;
4237 
4238 softnet_break:
4239 	sd->time_squeeze++;
4240 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
4241 	goto out;
4242 }
4243 
4244 struct netdev_upper {
4245 	struct net_device *dev;
4246 	bool master;
4247 	struct list_head list;
4248 	struct rcu_head rcu;
4249 	struct list_head search_list;
4250 };
4251 
__append_search_uppers(struct list_head * search_list,struct net_device * dev)4252 static void __append_search_uppers(struct list_head *search_list,
4253 				   struct net_device *dev)
4254 {
4255 	struct netdev_upper *upper;
4256 
4257 	list_for_each_entry(upper, &dev->upper_dev_list, list) {
4258 		/* check if this upper is not already in search list */
4259 		if (list_empty(&upper->search_list))
4260 			list_add_tail(&upper->search_list, search_list);
4261 	}
4262 }
4263 
__netdev_search_upper_dev(struct net_device * dev,struct net_device * upper_dev)4264 static bool __netdev_search_upper_dev(struct net_device *dev,
4265 				      struct net_device *upper_dev)
4266 {
4267 	LIST_HEAD(search_list);
4268 	struct netdev_upper *upper;
4269 	struct netdev_upper *tmp;
4270 	bool ret = false;
4271 
4272 	__append_search_uppers(&search_list, dev);
4273 	list_for_each_entry(upper, &search_list, search_list) {
4274 		if (upper->dev == upper_dev) {
4275 			ret = true;
4276 			break;
4277 		}
4278 		__append_search_uppers(&search_list, upper->dev);
4279 	}
4280 	list_for_each_entry_safe(upper, tmp, &search_list, search_list)
4281 		INIT_LIST_HEAD(&upper->search_list);
4282 	return ret;
4283 }
4284 
__netdev_find_upper(struct net_device * dev,struct net_device * upper_dev)4285 static struct netdev_upper *__netdev_find_upper(struct net_device *dev,
4286 						struct net_device *upper_dev)
4287 {
4288 	struct netdev_upper *upper;
4289 
4290 	list_for_each_entry(upper, &dev->upper_dev_list, list) {
4291 		if (upper->dev == upper_dev)
4292 			return upper;
4293 	}
4294 	return NULL;
4295 }
4296 
4297 /**
4298  * netdev_has_upper_dev - Check if device is linked to an upper device
4299  * @dev: device
4300  * @upper_dev: upper device to check
4301  *
4302  * Find out if a device is linked to specified upper device and return true
4303  * in case it is. Note that this checks only immediate upper device,
4304  * not through a complete stack of devices. The caller must hold the RTNL lock.
4305  */
netdev_has_upper_dev(struct net_device * dev,struct net_device * upper_dev)4306 bool netdev_has_upper_dev(struct net_device *dev,
4307 			  struct net_device *upper_dev)
4308 {
4309 	ASSERT_RTNL();
4310 
4311 	return __netdev_find_upper(dev, upper_dev);
4312 }
4313 EXPORT_SYMBOL(netdev_has_upper_dev);
4314 
4315 /**
4316  * netdev_has_any_upper_dev - Check if device is linked to some device
4317  * @dev: device
4318  *
4319  * Find out if a device is linked to an upper device and return true in case
4320  * it is. The caller must hold the RTNL lock.
4321  */
netdev_has_any_upper_dev(struct net_device * dev)4322 bool netdev_has_any_upper_dev(struct net_device *dev)
4323 {
4324 	ASSERT_RTNL();
4325 
4326 	return !list_empty(&dev->upper_dev_list);
4327 }
4328 EXPORT_SYMBOL(netdev_has_any_upper_dev);
4329 
4330 /**
4331  * netdev_master_upper_dev_get - Get master upper device
4332  * @dev: device
4333  *
4334  * Find a master upper device and return pointer to it or NULL in case
4335  * it's not there. The caller must hold the RTNL lock.
4336  */
netdev_master_upper_dev_get(struct net_device * dev)4337 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4338 {
4339 	struct netdev_upper *upper;
4340 
4341 	ASSERT_RTNL();
4342 
4343 	if (list_empty(&dev->upper_dev_list))
4344 		return NULL;
4345 
4346 	upper = list_first_entry(&dev->upper_dev_list,
4347 				 struct netdev_upper, list);
4348 	if (likely(upper->master))
4349 		return upper->dev;
4350 	return NULL;
4351 }
4352 EXPORT_SYMBOL(netdev_master_upper_dev_get);
4353 
4354 /**
4355  * netdev_master_upper_dev_get_rcu - Get master upper device
4356  * @dev: device
4357  *
4358  * Find a master upper device and return pointer to it or NULL in case
4359  * it's not there. The caller must hold the RCU read lock.
4360  */
netdev_master_upper_dev_get_rcu(struct net_device * dev)4361 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
4362 {
4363 	struct netdev_upper *upper;
4364 
4365 	upper = list_first_or_null_rcu(&dev->upper_dev_list,
4366 				       struct netdev_upper, list);
4367 	if (upper && likely(upper->master))
4368 		return upper->dev;
4369 	return NULL;
4370 }
4371 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
4372 
__netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev,bool master)4373 static int __netdev_upper_dev_link(struct net_device *dev,
4374 				   struct net_device *upper_dev, bool master)
4375 {
4376 	struct netdev_upper *upper;
4377 
4378 	ASSERT_RTNL();
4379 
4380 	if (dev == upper_dev)
4381 		return -EBUSY;
4382 
4383 	/* To prevent loops, check if dev is not upper device to upper_dev. */
4384 	if (__netdev_search_upper_dev(upper_dev, dev))
4385 		return -EBUSY;
4386 
4387 	if (__netdev_find_upper(dev, upper_dev))
4388 		return -EEXIST;
4389 
4390 	if (master && netdev_master_upper_dev_get(dev))
4391 		return -EBUSY;
4392 
4393 	upper = kmalloc(sizeof(*upper), GFP_KERNEL);
4394 	if (!upper)
4395 		return -ENOMEM;
4396 
4397 	upper->dev = upper_dev;
4398 	upper->master = master;
4399 	INIT_LIST_HEAD(&upper->search_list);
4400 
4401 	/* Ensure that master upper link is always the first item in list. */
4402 	if (master)
4403 		list_add_rcu(&upper->list, &dev->upper_dev_list);
4404 	else
4405 		list_add_tail_rcu(&upper->list, &dev->upper_dev_list);
4406 	dev_hold(upper_dev);
4407 
4408 	return 0;
4409 }
4410 
4411 /**
4412  * netdev_upper_dev_link - Add a link to the upper device
4413  * @dev: device
4414  * @upper_dev: new upper device
4415  *
4416  * Adds a link to device which is upper to this one. The caller must hold
4417  * the RTNL lock. On a failure a negative errno code is returned.
4418  * On success the reference counts are adjusted and the function
4419  * returns zero.
4420  */
netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev)4421 int netdev_upper_dev_link(struct net_device *dev,
4422 			  struct net_device *upper_dev)
4423 {
4424 	return __netdev_upper_dev_link(dev, upper_dev, false);
4425 }
4426 EXPORT_SYMBOL(netdev_upper_dev_link);
4427 
4428 /**
4429  * netdev_master_upper_dev_link - Add a master link to the upper device
4430  * @dev: device
4431  * @upper_dev: new upper device
4432  *
4433  * Adds a link to device which is upper to this one. In this case, only
4434  * one master upper device can be linked, although other non-master devices
4435  * might be linked as well. The caller must hold the RTNL lock.
4436  * On a failure a negative errno code is returned. On success the reference
4437  * counts are adjusted and the function returns zero.
4438  */
netdev_master_upper_dev_link(struct net_device * dev,struct net_device * upper_dev)4439 int netdev_master_upper_dev_link(struct net_device *dev,
4440 				 struct net_device *upper_dev)
4441 {
4442 	return __netdev_upper_dev_link(dev, upper_dev, true);
4443 }
4444 EXPORT_SYMBOL(netdev_master_upper_dev_link);
4445 
4446 /**
4447  * netdev_upper_dev_unlink - Removes a link to upper device
4448  * @dev: device
4449  * @upper_dev: new upper device
4450  *
4451  * Removes a link to device which is upper to this one. The caller must hold
4452  * the RTNL lock.
4453  */
netdev_upper_dev_unlink(struct net_device * dev,struct net_device * upper_dev)4454 void netdev_upper_dev_unlink(struct net_device *dev,
4455 			     struct net_device *upper_dev)
4456 {
4457 	struct netdev_upper *upper;
4458 
4459 	ASSERT_RTNL();
4460 
4461 	upper = __netdev_find_upper(dev, upper_dev);
4462 	if (!upper)
4463 		return;
4464 	list_del_rcu(&upper->list);
4465 	dev_put(upper_dev);
4466 	kfree_rcu(upper, rcu);
4467 }
4468 EXPORT_SYMBOL(netdev_upper_dev_unlink);
4469 
dev_change_rx_flags(struct net_device * dev,int flags)4470 static void dev_change_rx_flags(struct net_device *dev, int flags)
4471 {
4472 	const struct net_device_ops *ops = dev->netdev_ops;
4473 
4474 	if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4475 		ops->ndo_change_rx_flags(dev, flags);
4476 }
4477 
__dev_set_promiscuity(struct net_device * dev,int inc)4478 static int __dev_set_promiscuity(struct net_device *dev, int inc)
4479 {
4480 	unsigned int old_flags = dev->flags;
4481 	kuid_t uid;
4482 	kgid_t gid;
4483 
4484 	ASSERT_RTNL();
4485 
4486 	dev->flags |= IFF_PROMISC;
4487 	dev->promiscuity += inc;
4488 	if (dev->promiscuity == 0) {
4489 		/*
4490 		 * Avoid overflow.
4491 		 * If inc causes overflow, untouch promisc and return error.
4492 		 */
4493 		if (inc < 0)
4494 			dev->flags &= ~IFF_PROMISC;
4495 		else {
4496 			dev->promiscuity -= inc;
4497 			pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4498 				dev->name);
4499 			return -EOVERFLOW;
4500 		}
4501 	}
4502 	if (dev->flags != old_flags) {
4503 		pr_info("device %s %s promiscuous mode\n",
4504 			dev->name,
4505 			dev->flags & IFF_PROMISC ? "entered" : "left");
4506 		if (audit_enabled) {
4507 			current_uid_gid(&uid, &gid);
4508 			audit_log(current->audit_context, GFP_ATOMIC,
4509 				AUDIT_ANOM_PROMISCUOUS,
4510 				"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4511 				dev->name, (dev->flags & IFF_PROMISC),
4512 				(old_flags & IFF_PROMISC),
4513 				from_kuid(&init_user_ns, audit_get_loginuid(current)),
4514 				from_kuid(&init_user_ns, uid),
4515 				from_kgid(&init_user_ns, gid),
4516 				audit_get_sessionid(current));
4517 		}
4518 
4519 		dev_change_rx_flags(dev, IFF_PROMISC);
4520 	}
4521 	return 0;
4522 }
4523 
4524 /**
4525  *	dev_set_promiscuity	- update promiscuity count on a device
4526  *	@dev: device
4527  *	@inc: modifier
4528  *
4529  *	Add or remove promiscuity from a device. While the count in the device
4530  *	remains above zero the interface remains promiscuous. Once it hits zero
4531  *	the device reverts back to normal filtering operation. A negative inc
4532  *	value is used to drop promiscuity on the device.
4533  *	Return 0 if successful or a negative errno code on error.
4534  */
dev_set_promiscuity(struct net_device * dev,int inc)4535 int dev_set_promiscuity(struct net_device *dev, int inc)
4536 {
4537 	unsigned int old_flags = dev->flags;
4538 	int err;
4539 
4540 	err = __dev_set_promiscuity(dev, inc);
4541 	if (err < 0)
4542 		return err;
4543 	if (dev->flags != old_flags)
4544 		dev_set_rx_mode(dev);
4545 	return err;
4546 }
4547 EXPORT_SYMBOL(dev_set_promiscuity);
4548 
4549 /**
4550  *	dev_set_allmulti	- update allmulti count on a device
4551  *	@dev: device
4552  *	@inc: modifier
4553  *
4554  *	Add or remove reception of all multicast frames to a device. While the
4555  *	count in the device remains above zero the interface remains listening
4556  *	to all interfaces. Once it hits zero the device reverts back to normal
4557  *	filtering operation. A negative @inc value is used to drop the counter
4558  *	when releasing a resource needing all multicasts.
4559  *	Return 0 if successful or a negative errno code on error.
4560  */
4561 
dev_set_allmulti(struct net_device * dev,int inc)4562 int dev_set_allmulti(struct net_device *dev, int inc)
4563 {
4564 	unsigned int old_flags = dev->flags;
4565 
4566 	ASSERT_RTNL();
4567 
4568 	dev->flags |= IFF_ALLMULTI;
4569 	dev->allmulti += inc;
4570 	if (dev->allmulti == 0) {
4571 		/*
4572 		 * Avoid overflow.
4573 		 * If inc causes overflow, untouch allmulti and return error.
4574 		 */
4575 		if (inc < 0)
4576 			dev->flags &= ~IFF_ALLMULTI;
4577 		else {
4578 			dev->allmulti -= inc;
4579 			pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4580 				dev->name);
4581 			return -EOVERFLOW;
4582 		}
4583 	}
4584 	if (dev->flags ^ old_flags) {
4585 		dev_change_rx_flags(dev, IFF_ALLMULTI);
4586 		dev_set_rx_mode(dev);
4587 	}
4588 	return 0;
4589 }
4590 EXPORT_SYMBOL(dev_set_allmulti);
4591 
4592 /*
4593  *	Upload unicast and multicast address lists to device and
4594  *	configure RX filtering. When the device doesn't support unicast
4595  *	filtering it is put in promiscuous mode while unicast addresses
4596  *	are present.
4597  */
__dev_set_rx_mode(struct net_device * dev)4598 void __dev_set_rx_mode(struct net_device *dev)
4599 {
4600 	const struct net_device_ops *ops = dev->netdev_ops;
4601 
4602 	/* dev_open will call this function so the list will stay sane. */
4603 	if (!(dev->flags&IFF_UP))
4604 		return;
4605 
4606 	if (!netif_device_present(dev))
4607 		return;
4608 
4609 	if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
4610 		/* Unicast addresses changes may only happen under the rtnl,
4611 		 * therefore calling __dev_set_promiscuity here is safe.
4612 		 */
4613 		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4614 			__dev_set_promiscuity(dev, 1);
4615 			dev->uc_promisc = true;
4616 		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4617 			__dev_set_promiscuity(dev, -1);
4618 			dev->uc_promisc = false;
4619 		}
4620 	}
4621 
4622 	if (ops->ndo_set_rx_mode)
4623 		ops->ndo_set_rx_mode(dev);
4624 }
4625 
dev_set_rx_mode(struct net_device * dev)4626 void dev_set_rx_mode(struct net_device *dev)
4627 {
4628 	netif_addr_lock_bh(dev);
4629 	__dev_set_rx_mode(dev);
4630 	netif_addr_unlock_bh(dev);
4631 }
4632 
4633 /**
4634  *	dev_get_flags - get flags reported to userspace
4635  *	@dev: device
4636  *
4637  *	Get the combination of flag bits exported through APIs to userspace.
4638  */
dev_get_flags(const struct net_device * dev)4639 unsigned int dev_get_flags(const struct net_device *dev)
4640 {
4641 	unsigned int flags;
4642 
4643 	flags = (dev->flags & ~(IFF_PROMISC |
4644 				IFF_ALLMULTI |
4645 				IFF_RUNNING |
4646 				IFF_LOWER_UP |
4647 				IFF_DORMANT)) |
4648 		(dev->gflags & (IFF_PROMISC |
4649 				IFF_ALLMULTI));
4650 
4651 	if (netif_running(dev)) {
4652 		if (netif_oper_up(dev))
4653 			flags |= IFF_RUNNING;
4654 		if (netif_carrier_ok(dev))
4655 			flags |= IFF_LOWER_UP;
4656 		if (netif_dormant(dev))
4657 			flags |= IFF_DORMANT;
4658 	}
4659 
4660 	return flags;
4661 }
4662 EXPORT_SYMBOL(dev_get_flags);
4663 
__dev_change_flags(struct net_device * dev,unsigned int flags)4664 int __dev_change_flags(struct net_device *dev, unsigned int flags)
4665 {
4666 	unsigned int old_flags = dev->flags;
4667 	int ret;
4668 
4669 	ASSERT_RTNL();
4670 
4671 	/*
4672 	 *	Set the flags on our device.
4673 	 */
4674 
4675 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4676 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4677 			       IFF_AUTOMEDIA)) |
4678 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4679 				    IFF_ALLMULTI));
4680 
4681 	/*
4682 	 *	Load in the correct multicast list now the flags have changed.
4683 	 */
4684 
4685 	if ((old_flags ^ flags) & IFF_MULTICAST)
4686 		dev_change_rx_flags(dev, IFF_MULTICAST);
4687 
4688 	dev_set_rx_mode(dev);
4689 
4690 	/*
4691 	 *	Have we downed the interface. We handle IFF_UP ourselves
4692 	 *	according to user attempts to set it, rather than blindly
4693 	 *	setting it.
4694 	 */
4695 
4696 	ret = 0;
4697 	if ((old_flags ^ flags) & IFF_UP) {	/* Bit is different  ? */
4698 		ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4699 
4700 		if (!ret)
4701 			dev_set_rx_mode(dev);
4702 	}
4703 
4704 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
4705 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
4706 
4707 		dev->gflags ^= IFF_PROMISC;
4708 		dev_set_promiscuity(dev, inc);
4709 	}
4710 
4711 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4712 	   is important. Some (broken) drivers set IFF_PROMISC, when
4713 	   IFF_ALLMULTI is requested not asking us and not reporting.
4714 	 */
4715 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4716 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4717 
4718 		dev->gflags ^= IFF_ALLMULTI;
4719 		dev_set_allmulti(dev, inc);
4720 	}
4721 
4722 	return ret;
4723 }
4724 
__dev_notify_flags(struct net_device * dev,unsigned int old_flags)4725 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4726 {
4727 	unsigned int changes = dev->flags ^ old_flags;
4728 
4729 	if (changes & IFF_UP) {
4730 		if (dev->flags & IFF_UP)
4731 			call_netdevice_notifiers(NETDEV_UP, dev);
4732 		else
4733 			call_netdevice_notifiers(NETDEV_DOWN, dev);
4734 	}
4735 
4736 	if (dev->flags & IFF_UP &&
4737 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4738 		call_netdevice_notifiers(NETDEV_CHANGE, dev);
4739 }
4740 
4741 /**
4742  *	dev_change_flags - change device settings
4743  *	@dev: device
4744  *	@flags: device state flags
4745  *
4746  *	Change settings on device based state flags. The flags are
4747  *	in the userspace exported format.
4748  */
dev_change_flags(struct net_device * dev,unsigned int flags)4749 int dev_change_flags(struct net_device *dev, unsigned int flags)
4750 {
4751 	int ret;
4752 	unsigned int changes, old_flags = dev->flags;
4753 
4754 	ret = __dev_change_flags(dev, flags);
4755 	if (ret < 0)
4756 		return ret;
4757 
4758 	changes = old_flags ^ dev->flags;
4759 	if (changes)
4760 		rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4761 
4762 	__dev_notify_flags(dev, old_flags);
4763 	return ret;
4764 }
4765 EXPORT_SYMBOL(dev_change_flags);
4766 
4767 /**
4768  *	dev_set_mtu - Change maximum transfer unit
4769  *	@dev: device
4770  *	@new_mtu: new transfer unit
4771  *
4772  *	Change the maximum transfer size of the network device.
4773  */
dev_set_mtu(struct net_device * dev,int new_mtu)4774 int dev_set_mtu(struct net_device *dev, int new_mtu)
4775 {
4776 	const struct net_device_ops *ops = dev->netdev_ops;
4777 	int err;
4778 
4779 	if (new_mtu == dev->mtu)
4780 		return 0;
4781 
4782 	/*	MTU must be positive.	 */
4783 	if (new_mtu < 0)
4784 		return -EINVAL;
4785 
4786 	if (!netif_device_present(dev))
4787 		return -ENODEV;
4788 
4789 	err = 0;
4790 	if (ops->ndo_change_mtu)
4791 		err = ops->ndo_change_mtu(dev, new_mtu);
4792 	else
4793 		dev->mtu = new_mtu;
4794 
4795 	if (!err)
4796 		call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4797 	return err;
4798 }
4799 EXPORT_SYMBOL(dev_set_mtu);
4800 
4801 /**
4802  *	dev_set_group - Change group this device belongs to
4803  *	@dev: device
4804  *	@new_group: group this device should belong to
4805  */
dev_set_group(struct net_device * dev,int new_group)4806 void dev_set_group(struct net_device *dev, int new_group)
4807 {
4808 	dev->group = new_group;
4809 }
4810 EXPORT_SYMBOL(dev_set_group);
4811 
4812 /**
4813  *	dev_set_mac_address - Change Media Access Control Address
4814  *	@dev: device
4815  *	@sa: new address
4816  *
4817  *	Change the hardware (MAC) address of the device
4818  */
dev_set_mac_address(struct net_device * dev,struct sockaddr * sa)4819 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4820 {
4821 	const struct net_device_ops *ops = dev->netdev_ops;
4822 	int err;
4823 
4824 	if (!ops->ndo_set_mac_address)
4825 		return -EOPNOTSUPP;
4826 	if (sa->sa_family != dev->type)
4827 		return -EINVAL;
4828 	if (!netif_device_present(dev))
4829 		return -ENODEV;
4830 	err = ops->ndo_set_mac_address(dev, sa);
4831 	if (err)
4832 		return err;
4833 	dev->addr_assign_type = NET_ADDR_SET;
4834 	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4835 	add_device_randomness(dev->dev_addr, dev->addr_len);
4836 	return 0;
4837 }
4838 EXPORT_SYMBOL(dev_set_mac_address);
4839 
4840 /**
4841  *	dev_change_carrier - Change device carrier
4842  *	@dev: device
4843  *	@new_carrier: new value
4844  *
4845  *	Change device carrier
4846  */
dev_change_carrier(struct net_device * dev,bool new_carrier)4847 int dev_change_carrier(struct net_device *dev, bool new_carrier)
4848 {
4849 	const struct net_device_ops *ops = dev->netdev_ops;
4850 
4851 	if (!ops->ndo_change_carrier)
4852 		return -EOPNOTSUPP;
4853 	if (!netif_device_present(dev))
4854 		return -ENODEV;
4855 	return ops->ndo_change_carrier(dev, new_carrier);
4856 }
4857 EXPORT_SYMBOL(dev_change_carrier);
4858 
4859 /**
4860  *	dev_new_index	-	allocate an ifindex
4861  *	@net: the applicable net namespace
4862  *
4863  *	Returns a suitable unique value for a new device interface
4864  *	number.  The caller must hold the rtnl semaphore or the
4865  *	dev_base_lock to be sure it remains unique.
4866  */
dev_new_index(struct net * net)4867 static int dev_new_index(struct net *net)
4868 {
4869 	int ifindex = net->ifindex;
4870 	for (;;) {
4871 		if (++ifindex <= 0)
4872 			ifindex = 1;
4873 		if (!__dev_get_by_index(net, ifindex))
4874 			return net->ifindex = ifindex;
4875 	}
4876 }
4877 
4878 /* Delayed registration/unregisteration */
4879 static LIST_HEAD(net_todo_list);
4880 
net_set_todo(struct net_device * dev)4881 static void net_set_todo(struct net_device *dev)
4882 {
4883 	list_add_tail(&dev->todo_list, &net_todo_list);
4884 }
4885 
rollback_registered_many(struct list_head * head)4886 static void rollback_registered_many(struct list_head *head)
4887 {
4888 	struct net_device *dev, *tmp;
4889 
4890 	BUG_ON(dev_boot_phase);
4891 	ASSERT_RTNL();
4892 
4893 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
4894 		/* Some devices call without registering
4895 		 * for initialization unwind. Remove those
4896 		 * devices and proceed with the remaining.
4897 		 */
4898 		if (dev->reg_state == NETREG_UNINITIALIZED) {
4899 			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
4900 				 dev->name, dev);
4901 
4902 			WARN_ON(1);
4903 			list_del(&dev->unreg_list);
4904 			continue;
4905 		}
4906 		dev->dismantle = true;
4907 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
4908 	}
4909 
4910 	/* If device is running, close it first. */
4911 	dev_close_many(head);
4912 
4913 	list_for_each_entry(dev, head, unreg_list) {
4914 		/* And unlink it from device chain. */
4915 		unlist_netdevice(dev);
4916 
4917 		dev->reg_state = NETREG_UNREGISTERING;
4918 	}
4919 
4920 	synchronize_net();
4921 
4922 	list_for_each_entry(dev, head, unreg_list) {
4923 		/* Shutdown queueing discipline. */
4924 		dev_shutdown(dev);
4925 
4926 
4927 		/* Notify protocols, that we are about to destroy
4928 		   this device. They should clean all the things.
4929 		*/
4930 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4931 
4932 		if (!dev->rtnl_link_ops ||
4933 		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4934 			rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
4935 
4936 		/*
4937 		 *	Flush the unicast and multicast chains
4938 		 */
4939 		dev_uc_flush(dev);
4940 		dev_mc_flush(dev);
4941 
4942 		if (dev->netdev_ops->ndo_uninit)
4943 			dev->netdev_ops->ndo_uninit(dev);
4944 
4945 		/* Notifier chain MUST detach us all upper devices. */
4946 		WARN_ON(netdev_has_any_upper_dev(dev));
4947 
4948 		/* Remove entries from kobject tree */
4949 		netdev_unregister_kobject(dev);
4950 #ifdef CONFIG_XPS
4951 		/* Remove XPS queueing entries */
4952 		netif_reset_xps_queues_gt(dev, 0);
4953 #endif
4954 	}
4955 
4956 	synchronize_net();
4957 
4958 	list_for_each_entry(dev, head, unreg_list)
4959 		dev_put(dev);
4960 }
4961 
rollback_registered(struct net_device * dev)4962 static void rollback_registered(struct net_device *dev)
4963 {
4964 	LIST_HEAD(single);
4965 
4966 	list_add(&dev->unreg_list, &single);
4967 	rollback_registered_many(&single);
4968 	list_del(&single);
4969 }
4970 
netdev_fix_features(struct net_device * dev,netdev_features_t features)4971 static netdev_features_t netdev_fix_features(struct net_device *dev,
4972 	netdev_features_t features)
4973 {
4974 	/* Fix illegal checksum combinations */
4975 	if ((features & NETIF_F_HW_CSUM) &&
4976 	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4977 		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
4978 		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4979 	}
4980 
4981 	/* TSO requires that SG is present as well. */
4982 	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
4983 		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
4984 		features &= ~NETIF_F_ALL_TSO;
4985 	}
4986 
4987 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
4988 					!(features & NETIF_F_IP_CSUM)) {
4989 		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
4990 		features &= ~NETIF_F_TSO;
4991 		features &= ~NETIF_F_TSO_ECN;
4992 	}
4993 
4994 	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
4995 					 !(features & NETIF_F_IPV6_CSUM)) {
4996 		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
4997 		features &= ~NETIF_F_TSO6;
4998 	}
4999 
5000 	/* TSO ECN requires that TSO is present as well. */
5001 	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5002 		features &= ~NETIF_F_TSO_ECN;
5003 
5004 	/* Software GSO depends on SG. */
5005 	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
5006 		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
5007 		features &= ~NETIF_F_GSO;
5008 	}
5009 
5010 	/* UFO needs SG and checksumming */
5011 	if (features & NETIF_F_UFO) {
5012 		/* maybe split UFO into V4 and V6? */
5013 		if (!((features & NETIF_F_GEN_CSUM) ||
5014 		    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5015 			    == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5016 			netdev_dbg(dev,
5017 				"Dropping NETIF_F_UFO since no checksum offload features.\n");
5018 			features &= ~NETIF_F_UFO;
5019 		}
5020 
5021 		if (!(features & NETIF_F_SG)) {
5022 			netdev_dbg(dev,
5023 				"Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
5024 			features &= ~NETIF_F_UFO;
5025 		}
5026 	}
5027 
5028 	return features;
5029 }
5030 
__netdev_update_features(struct net_device * dev)5031 int __netdev_update_features(struct net_device *dev)
5032 {
5033 	netdev_features_t features;
5034 	int err = 0;
5035 
5036 	ASSERT_RTNL();
5037 
5038 	features = netdev_get_wanted_features(dev);
5039 
5040 	if (dev->netdev_ops->ndo_fix_features)
5041 		features = dev->netdev_ops->ndo_fix_features(dev, features);
5042 
5043 	/* driver might be less strict about feature dependencies */
5044 	features = netdev_fix_features(dev, features);
5045 
5046 	if (dev->features == features)
5047 		return 0;
5048 
5049 	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5050 		&dev->features, &features);
5051 
5052 	if (dev->netdev_ops->ndo_set_features)
5053 		err = dev->netdev_ops->ndo_set_features(dev, features);
5054 
5055 	if (unlikely(err < 0)) {
5056 		netdev_err(dev,
5057 			"set_features() failed (%d); wanted %pNF, left %pNF\n",
5058 			err, &features, &dev->features);
5059 		return -1;
5060 	}
5061 
5062 	if (!err)
5063 		dev->features = features;
5064 
5065 	return 1;
5066 }
5067 
5068 /**
5069  *	netdev_update_features - recalculate device features
5070  *	@dev: the device to check
5071  *
5072  *	Recalculate dev->features set and send notifications if it
5073  *	has changed. Should be called after driver or hardware dependent
5074  *	conditions might have changed that influence the features.
5075  */
netdev_update_features(struct net_device * dev)5076 void netdev_update_features(struct net_device *dev)
5077 {
5078 	if (__netdev_update_features(dev))
5079 		netdev_features_change(dev);
5080 }
5081 EXPORT_SYMBOL(netdev_update_features);
5082 
5083 /**
5084  *	netdev_change_features - recalculate device features
5085  *	@dev: the device to check
5086  *
5087  *	Recalculate dev->features set and send notifications even
5088  *	if they have not changed. Should be called instead of
5089  *	netdev_update_features() if also dev->vlan_features might
5090  *	have changed to allow the changes to be propagated to stacked
5091  *	VLAN devices.
5092  */
netdev_change_features(struct net_device * dev)5093 void netdev_change_features(struct net_device *dev)
5094 {
5095 	__netdev_update_features(dev);
5096 	netdev_features_change(dev);
5097 }
5098 EXPORT_SYMBOL(netdev_change_features);
5099 
5100 /**
5101  *	netif_stacked_transfer_operstate -	transfer operstate
5102  *	@rootdev: the root or lower level device to transfer state from
5103  *	@dev: the device to transfer operstate to
5104  *
5105  *	Transfer operational state from root to device. This is normally
5106  *	called when a stacking relationship exists between the root
5107  *	device and the device(a leaf device).
5108  */
netif_stacked_transfer_operstate(const struct net_device * rootdev,struct net_device * dev)5109 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5110 					struct net_device *dev)
5111 {
5112 	if (rootdev->operstate == IF_OPER_DORMANT)
5113 		netif_dormant_on(dev);
5114 	else
5115 		netif_dormant_off(dev);
5116 
5117 	if (netif_carrier_ok(rootdev)) {
5118 		if (!netif_carrier_ok(dev))
5119 			netif_carrier_on(dev);
5120 	} else {
5121 		if (netif_carrier_ok(dev))
5122 			netif_carrier_off(dev);
5123 	}
5124 }
5125 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5126 
5127 #ifdef CONFIG_RPS
netif_alloc_rx_queues(struct net_device * dev)5128 static int netif_alloc_rx_queues(struct net_device *dev)
5129 {
5130 	unsigned int i, count = dev->num_rx_queues;
5131 	struct netdev_rx_queue *rx;
5132 
5133 	BUG_ON(count < 1);
5134 
5135 	rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5136 	if (!rx)
5137 		return -ENOMEM;
5138 
5139 	dev->_rx = rx;
5140 
5141 	for (i = 0; i < count; i++)
5142 		rx[i].dev = dev;
5143 	return 0;
5144 }
5145 #endif
5146 
netdev_init_one_queue(struct net_device * dev,struct netdev_queue * queue,void * _unused)5147 static void netdev_init_one_queue(struct net_device *dev,
5148 				  struct netdev_queue *queue, void *_unused)
5149 {
5150 	/* Initialize queue lock */
5151 	spin_lock_init(&queue->_xmit_lock);
5152 	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5153 	queue->xmit_lock_owner = -1;
5154 	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5155 	queue->dev = dev;
5156 #ifdef CONFIG_BQL
5157 	dql_init(&queue->dql, HZ);
5158 #endif
5159 }
5160 
netif_alloc_netdev_queues(struct net_device * dev)5161 static int netif_alloc_netdev_queues(struct net_device *dev)
5162 {
5163 	unsigned int count = dev->num_tx_queues;
5164 	struct netdev_queue *tx;
5165 
5166 	BUG_ON(count < 1);
5167 
5168 	tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5169 	if (!tx)
5170 		return -ENOMEM;
5171 
5172 	dev->_tx = tx;
5173 
5174 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5175 	spin_lock_init(&dev->tx_global_lock);
5176 
5177 	return 0;
5178 }
5179 
5180 /**
5181  *	register_netdevice	- register a network device
5182  *	@dev: device to register
5183  *
5184  *	Take a completed network device structure and add it to the kernel
5185  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5186  *	chain. 0 is returned on success. A negative errno code is returned
5187  *	on a failure to set up the device, or if the name is a duplicate.
5188  *
5189  *	Callers must hold the rtnl semaphore. You may want
5190  *	register_netdev() instead of this.
5191  *
5192  *	BUGS:
5193  *	The locking appears insufficient to guarantee two parallel registers
5194  *	will not get the same name.
5195  */
5196 
register_netdevice(struct net_device * dev)5197 int register_netdevice(struct net_device *dev)
5198 {
5199 	int ret;
5200 	struct net *net = dev_net(dev);
5201 
5202 	BUG_ON(dev_boot_phase);
5203 	ASSERT_RTNL();
5204 
5205 	might_sleep();
5206 
5207 	/* When net_device's are persistent, this will be fatal. */
5208 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5209 	BUG_ON(!net);
5210 
5211 	spin_lock_init(&dev->addr_list_lock);
5212 	netdev_set_addr_lockdep_class(dev);
5213 
5214 	dev->iflink = -1;
5215 
5216 	ret = dev_get_valid_name(net, dev, dev->name);
5217 	if (ret < 0)
5218 		goto out;
5219 
5220 	/* Init, if this function is available */
5221 	if (dev->netdev_ops->ndo_init) {
5222 		ret = dev->netdev_ops->ndo_init(dev);
5223 		if (ret) {
5224 			if (ret > 0)
5225 				ret = -EIO;
5226 			goto out;
5227 		}
5228 	}
5229 
5230 	if (((dev->hw_features | dev->features) &
5231 	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
5232 	    (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
5233 	     !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
5234 		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
5235 		ret = -EINVAL;
5236 		goto err_uninit;
5237 	}
5238 
5239 	ret = -EBUSY;
5240 	if (!dev->ifindex)
5241 		dev->ifindex = dev_new_index(net);
5242 	else if (__dev_get_by_index(net, dev->ifindex))
5243 		goto err_uninit;
5244 
5245 	if (dev->iflink == -1)
5246 		dev->iflink = dev->ifindex;
5247 
5248 	/* Transfer changeable features to wanted_features and enable
5249 	 * software offloads (GSO and GRO).
5250 	 */
5251 	dev->hw_features |= NETIF_F_SOFT_FEATURES;
5252 	dev->features |= NETIF_F_SOFT_FEATURES;
5253 	dev->wanted_features = dev->features & dev->hw_features;
5254 
5255 	/* Turn on no cache copy if HW is doing checksum */
5256 	if (!(dev->flags & IFF_LOOPBACK)) {
5257 		dev->hw_features |= NETIF_F_NOCACHE_COPY;
5258 		if (dev->features & NETIF_F_ALL_CSUM) {
5259 			dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5260 			dev->features |= NETIF_F_NOCACHE_COPY;
5261 		}
5262 	}
5263 
5264 	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
5265 	 */
5266 	dev->vlan_features |= NETIF_F_HIGHDMA;
5267 
5268 	/* Make NETIF_F_SG inheritable to tunnel devices.
5269 	 */
5270 	dev->hw_enc_features |= NETIF_F_SG;
5271 
5272 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5273 	ret = notifier_to_errno(ret);
5274 	if (ret)
5275 		goto err_uninit;
5276 
5277 	ret = netdev_register_kobject(dev);
5278 	if (ret)
5279 		goto err_uninit;
5280 	dev->reg_state = NETREG_REGISTERED;
5281 
5282 	__netdev_update_features(dev);
5283 
5284 	/*
5285 	 *	Default initial state at registry is that the
5286 	 *	device is present.
5287 	 */
5288 
5289 	set_bit(__LINK_STATE_PRESENT, &dev->state);
5290 
5291 	linkwatch_init_dev(dev);
5292 
5293 	dev_init_scheduler(dev);
5294 	dev_hold(dev);
5295 	list_netdevice(dev);
5296 	add_device_randomness(dev->dev_addr, dev->addr_len);
5297 
5298 	/* If the device has permanent device address, driver should
5299 	 * set dev_addr and also addr_assign_type should be set to
5300 	 * NET_ADDR_PERM (default value).
5301 	 */
5302 	if (dev->addr_assign_type == NET_ADDR_PERM)
5303 		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
5304 
5305 	/* Notify protocols, that a new device appeared. */
5306 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5307 	ret = notifier_to_errno(ret);
5308 	if (ret) {
5309 		rollback_registered(dev);
5310 		dev->reg_state = NETREG_UNREGISTERED;
5311 	}
5312 	/*
5313 	 *	Prevent userspace races by waiting until the network
5314 	 *	device is fully setup before sending notifications.
5315 	 */
5316 	if (!dev->rtnl_link_ops ||
5317 	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5318 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5319 
5320 out:
5321 	return ret;
5322 
5323 err_uninit:
5324 	if (dev->netdev_ops->ndo_uninit)
5325 		dev->netdev_ops->ndo_uninit(dev);
5326 	goto out;
5327 }
5328 EXPORT_SYMBOL(register_netdevice);
5329 
5330 /**
5331  *	init_dummy_netdev	- init a dummy network device for NAPI
5332  *	@dev: device to init
5333  *
5334  *	This takes a network device structure and initialize the minimum
5335  *	amount of fields so it can be used to schedule NAPI polls without
5336  *	registering a full blown interface. This is to be used by drivers
5337  *	that need to tie several hardware interfaces to a single NAPI
5338  *	poll scheduler due to HW limitations.
5339  */
init_dummy_netdev(struct net_device * dev)5340 int init_dummy_netdev(struct net_device *dev)
5341 {
5342 	/* Clear everything. Note we don't initialize spinlocks
5343 	 * are they aren't supposed to be taken by any of the
5344 	 * NAPI code and this dummy netdev is supposed to be
5345 	 * only ever used for NAPI polls
5346 	 */
5347 	memset(dev, 0, sizeof(struct net_device));
5348 
5349 	/* make sure we BUG if trying to hit standard
5350 	 * register/unregister code path
5351 	 */
5352 	dev->reg_state = NETREG_DUMMY;
5353 
5354 	/* NAPI wants this */
5355 	INIT_LIST_HEAD(&dev->napi_list);
5356 
5357 	/* a dummy interface is started by default */
5358 	set_bit(__LINK_STATE_PRESENT, &dev->state);
5359 	set_bit(__LINK_STATE_START, &dev->state);
5360 
5361 	/* Note : We dont allocate pcpu_refcnt for dummy devices,
5362 	 * because users of this 'device' dont need to change
5363 	 * its refcount.
5364 	 */
5365 
5366 	return 0;
5367 }
5368 EXPORT_SYMBOL_GPL(init_dummy_netdev);
5369 
5370 
5371 /**
5372  *	register_netdev	- register a network device
5373  *	@dev: device to register
5374  *
5375  *	Take a completed network device structure and add it to the kernel
5376  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5377  *	chain. 0 is returned on success. A negative errno code is returned
5378  *	on a failure to set up the device, or if the name is a duplicate.
5379  *
5380  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
5381  *	and expands the device name if you passed a format string to
5382  *	alloc_netdev.
5383  */
register_netdev(struct net_device * dev)5384 int register_netdev(struct net_device *dev)
5385 {
5386 	int err;
5387 
5388 	rtnl_lock();
5389 	err = register_netdevice(dev);
5390 	rtnl_unlock();
5391 	return err;
5392 }
5393 EXPORT_SYMBOL(register_netdev);
5394 
netdev_refcnt_read(const struct net_device * dev)5395 int netdev_refcnt_read(const struct net_device *dev)
5396 {
5397 	int i, refcnt = 0;
5398 
5399 	for_each_possible_cpu(i)
5400 		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5401 	return refcnt;
5402 }
5403 EXPORT_SYMBOL(netdev_refcnt_read);
5404 
5405 /**
5406  * netdev_wait_allrefs - wait until all references are gone.
5407  * @dev: target net_device
5408  *
5409  * This is called when unregistering network devices.
5410  *
5411  * Any protocol or device that holds a reference should register
5412  * for netdevice notification, and cleanup and put back the
5413  * reference if they receive an UNREGISTER event.
5414  * We can get stuck here if buggy protocols don't correctly
5415  * call dev_put.
5416  */
netdev_wait_allrefs(struct net_device * dev)5417 static void netdev_wait_allrefs(struct net_device *dev)
5418 {
5419 	unsigned long rebroadcast_time, warning_time;
5420 	int refcnt;
5421 
5422 	linkwatch_forget_dev(dev);
5423 
5424 	rebroadcast_time = warning_time = jiffies;
5425 	refcnt = netdev_refcnt_read(dev);
5426 
5427 	while (refcnt != 0) {
5428 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5429 			rtnl_lock();
5430 
5431 			/* Rebroadcast unregister notification */
5432 			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5433 
5434 			__rtnl_unlock();
5435 			rcu_barrier();
5436 			rtnl_lock();
5437 
5438 			call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5439 			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5440 				     &dev->state)) {
5441 				/* We must not have linkwatch events
5442 				 * pending on unregister. If this
5443 				 * happens, we simply run the queue
5444 				 * unscheduled, resulting in a noop
5445 				 * for this device.
5446 				 */
5447 				linkwatch_run_queue();
5448 			}
5449 
5450 			__rtnl_unlock();
5451 
5452 			rebroadcast_time = jiffies;
5453 		}
5454 
5455 		msleep(250);
5456 
5457 		refcnt = netdev_refcnt_read(dev);
5458 
5459 		if (time_after(jiffies, warning_time + 10 * HZ)) {
5460 			pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5461 				 dev->name, refcnt);
5462 			warning_time = jiffies;
5463 		}
5464 	}
5465 }
5466 
5467 /* The sequence is:
5468  *
5469  *	rtnl_lock();
5470  *	...
5471  *	register_netdevice(x1);
5472  *	register_netdevice(x2);
5473  *	...
5474  *	unregister_netdevice(y1);
5475  *	unregister_netdevice(y2);
5476  *      ...
5477  *	rtnl_unlock();
5478  *	free_netdev(y1);
5479  *	free_netdev(y2);
5480  *
5481  * We are invoked by rtnl_unlock().
5482  * This allows us to deal with problems:
5483  * 1) We can delete sysfs objects which invoke hotplug
5484  *    without deadlocking with linkwatch via keventd.
5485  * 2) Since we run with the RTNL semaphore not held, we can sleep
5486  *    safely in order to wait for the netdev refcnt to drop to zero.
5487  *
5488  * We must not return until all unregister events added during
5489  * the interval the lock was held have been completed.
5490  */
netdev_run_todo(void)5491 void netdev_run_todo(void)
5492 {
5493 	struct list_head list;
5494 
5495 	/* Snapshot list, allow later requests */
5496 	list_replace_init(&net_todo_list, &list);
5497 
5498 	__rtnl_unlock();
5499 
5500 
5501 	/* Wait for rcu callbacks to finish before next phase */
5502 	if (!list_empty(&list))
5503 		rcu_barrier();
5504 
5505 	while (!list_empty(&list)) {
5506 		struct net_device *dev
5507 			= list_first_entry(&list, struct net_device, todo_list);
5508 		list_del(&dev->todo_list);
5509 
5510 		rtnl_lock();
5511 		call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5512 		__rtnl_unlock();
5513 
5514 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5515 			pr_err("network todo '%s' but state %d\n",
5516 			       dev->name, dev->reg_state);
5517 			dump_stack();
5518 			continue;
5519 		}
5520 
5521 		dev->reg_state = NETREG_UNREGISTERED;
5522 
5523 		on_each_cpu(flush_backlog, dev, 1);
5524 
5525 		netdev_wait_allrefs(dev);
5526 
5527 		/* paranoia */
5528 		BUG_ON(netdev_refcnt_read(dev));
5529 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
5530 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
5531 		WARN_ON(dev->dn_ptr);
5532 
5533 		if (dev->destructor)
5534 			dev->destructor(dev);
5535 
5536 		/* Free network device */
5537 		kobject_put(&dev->dev.kobj);
5538 	}
5539 }
5540 
5541 /* Convert net_device_stats to rtnl_link_stats64.  They have the same
5542  * fields in the same order, with only the type differing.
5543  */
netdev_stats_to_stats64(struct rtnl_link_stats64 * stats64,const struct net_device_stats * netdev_stats)5544 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5545 			     const struct net_device_stats *netdev_stats)
5546 {
5547 #if BITS_PER_LONG == 64
5548 	BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5549 	memcpy(stats64, netdev_stats, sizeof(*stats64));
5550 #else
5551 	size_t i, n = sizeof(*stats64) / sizeof(u64);
5552 	const unsigned long *src = (const unsigned long *)netdev_stats;
5553 	u64 *dst = (u64 *)stats64;
5554 
5555 	BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5556 		     sizeof(*stats64) / sizeof(u64));
5557 	for (i = 0; i < n; i++)
5558 		dst[i] = src[i];
5559 #endif
5560 }
5561 EXPORT_SYMBOL(netdev_stats_to_stats64);
5562 
5563 /**
5564  *	dev_get_stats	- get network device statistics
5565  *	@dev: device to get statistics from
5566  *	@storage: place to store stats
5567  *
5568  *	Get network statistics from device. Return @storage.
5569  *	The device driver may provide its own method by setting
5570  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5571  *	otherwise the internal statistics structure is used.
5572  */
dev_get_stats(struct net_device * dev,struct rtnl_link_stats64 * storage)5573 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5574 					struct rtnl_link_stats64 *storage)
5575 {
5576 	const struct net_device_ops *ops = dev->netdev_ops;
5577 
5578 	if (ops->ndo_get_stats64) {
5579 		memset(storage, 0, sizeof(*storage));
5580 		ops->ndo_get_stats64(dev, storage);
5581 	} else if (ops->ndo_get_stats) {
5582 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5583 	} else {
5584 		netdev_stats_to_stats64(storage, &dev->stats);
5585 	}
5586 	storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5587 	return storage;
5588 }
5589 EXPORT_SYMBOL(dev_get_stats);
5590 
dev_ingress_queue_create(struct net_device * dev)5591 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5592 {
5593 	struct netdev_queue *queue = dev_ingress_queue(dev);
5594 
5595 #ifdef CONFIG_NET_CLS_ACT
5596 	if (queue)
5597 		return queue;
5598 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5599 	if (!queue)
5600 		return NULL;
5601 	netdev_init_one_queue(dev, queue, NULL);
5602 	queue->qdisc = &noop_qdisc;
5603 	queue->qdisc_sleeping = &noop_qdisc;
5604 	rcu_assign_pointer(dev->ingress_queue, queue);
5605 #endif
5606 	return queue;
5607 }
5608 
5609 static const struct ethtool_ops default_ethtool_ops;
5610 
netdev_set_default_ethtool_ops(struct net_device * dev,const struct ethtool_ops * ops)5611 void netdev_set_default_ethtool_ops(struct net_device *dev,
5612 				    const struct ethtool_ops *ops)
5613 {
5614 	if (dev->ethtool_ops == &default_ethtool_ops)
5615 		dev->ethtool_ops = ops;
5616 }
5617 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
5618 
5619 /**
5620  *	alloc_netdev_mqs - allocate network device
5621  *	@sizeof_priv:	size of private data to allocate space for
5622  *	@name:		device name format string
5623  *	@setup:		callback to initialize device
5624  *	@txqs:		the number of TX subqueues to allocate
5625  *	@rxqs:		the number of RX subqueues to allocate
5626  *
5627  *	Allocates a struct net_device with private data area for driver use
5628  *	and performs basic initialization.  Also allocates subquue structs
5629  *	for each queue on the device.
5630  */
alloc_netdev_mqs(int sizeof_priv,const char * name,void (* setup)(struct net_device *),unsigned int txqs,unsigned int rxqs)5631 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5632 		void (*setup)(struct net_device *),
5633 		unsigned int txqs, unsigned int rxqs)
5634 {
5635 	struct net_device *dev;
5636 	size_t alloc_size;
5637 	struct net_device *p;
5638 
5639 	BUG_ON(strlen(name) >= sizeof(dev->name));
5640 
5641 	if (txqs < 1) {
5642 		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
5643 		return NULL;
5644 	}
5645 
5646 #ifdef CONFIG_RPS
5647 	if (rxqs < 1) {
5648 		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
5649 		return NULL;
5650 	}
5651 #endif
5652 
5653 	alloc_size = sizeof(struct net_device);
5654 	if (sizeof_priv) {
5655 		/* ensure 32-byte alignment of private area */
5656 		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5657 		alloc_size += sizeof_priv;
5658 	}
5659 	/* ensure 32-byte alignment of whole construct */
5660 	alloc_size += NETDEV_ALIGN - 1;
5661 
5662 	p = kzalloc(alloc_size, GFP_KERNEL);
5663 	if (!p)
5664 		return NULL;
5665 
5666 	dev = PTR_ALIGN(p, NETDEV_ALIGN);
5667 	dev->padded = (char *)dev - (char *)p;
5668 
5669 	dev->pcpu_refcnt = alloc_percpu(int);
5670 	if (!dev->pcpu_refcnt)
5671 		goto free_p;
5672 
5673 	if (dev_addr_init(dev))
5674 		goto free_pcpu;
5675 
5676 	dev_mc_init(dev);
5677 	dev_uc_init(dev);
5678 
5679 	dev_net_set(dev, &init_net);
5680 
5681 	dev->gso_max_size = GSO_MAX_SIZE;
5682 	dev->gso_max_segs = GSO_MAX_SEGS;
5683 
5684 	INIT_LIST_HEAD(&dev->napi_list);
5685 	INIT_LIST_HEAD(&dev->unreg_list);
5686 	INIT_LIST_HEAD(&dev->link_watch_list);
5687 	INIT_LIST_HEAD(&dev->upper_dev_list);
5688 	dev->priv_flags = IFF_XMIT_DST_RELEASE;
5689 	setup(dev);
5690 
5691 	dev->num_tx_queues = txqs;
5692 	dev->real_num_tx_queues = txqs;
5693 	if (netif_alloc_netdev_queues(dev))
5694 		goto free_all;
5695 
5696 #ifdef CONFIG_RPS
5697 	dev->num_rx_queues = rxqs;
5698 	dev->real_num_rx_queues = rxqs;
5699 	if (netif_alloc_rx_queues(dev))
5700 		goto free_all;
5701 #endif
5702 
5703 	strcpy(dev->name, name);
5704 	dev->group = INIT_NETDEV_GROUP;
5705 	if (!dev->ethtool_ops)
5706 		dev->ethtool_ops = &default_ethtool_ops;
5707 	return dev;
5708 
5709 free_all:
5710 	free_netdev(dev);
5711 	return NULL;
5712 
5713 free_pcpu:
5714 	free_percpu(dev->pcpu_refcnt);
5715 	kfree(dev->_tx);
5716 #ifdef CONFIG_RPS
5717 	kfree(dev->_rx);
5718 #endif
5719 
5720 free_p:
5721 	kfree(p);
5722 	return NULL;
5723 }
5724 EXPORT_SYMBOL(alloc_netdev_mqs);
5725 
5726 /**
5727  *	free_netdev - free network device
5728  *	@dev: device
5729  *
5730  *	This function does the last stage of destroying an allocated device
5731  * 	interface. The reference to the device object is released.
5732  *	If this is the last reference then it will be freed.
5733  */
free_netdev(struct net_device * dev)5734 void free_netdev(struct net_device *dev)
5735 {
5736 	struct napi_struct *p, *n;
5737 
5738 	release_net(dev_net(dev));
5739 
5740 	kfree(dev->_tx);
5741 #ifdef CONFIG_RPS
5742 	kfree(dev->_rx);
5743 #endif
5744 
5745 	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
5746 
5747 	/* Flush device addresses */
5748 	dev_addr_flush(dev);
5749 
5750 	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5751 		netif_napi_del(p);
5752 
5753 	free_percpu(dev->pcpu_refcnt);
5754 	dev->pcpu_refcnt = NULL;
5755 
5756 	/*  Compatibility with error handling in drivers */
5757 	if (dev->reg_state == NETREG_UNINITIALIZED) {
5758 		kfree((char *)dev - dev->padded);
5759 		return;
5760 	}
5761 
5762 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5763 	dev->reg_state = NETREG_RELEASED;
5764 
5765 	/* will free via device release */
5766 	put_device(&dev->dev);
5767 }
5768 EXPORT_SYMBOL(free_netdev);
5769 
5770 /**
5771  *	synchronize_net -  Synchronize with packet receive processing
5772  *
5773  *	Wait for packets currently being received to be done.
5774  *	Does not block later packets from starting.
5775  */
synchronize_net(void)5776 void synchronize_net(void)
5777 {
5778 	might_sleep();
5779 	if (rtnl_is_locked())
5780 		synchronize_rcu_expedited();
5781 	else
5782 		synchronize_rcu();
5783 }
5784 EXPORT_SYMBOL(synchronize_net);
5785 
5786 /**
5787  *	unregister_netdevice_queue - remove device from the kernel
5788  *	@dev: device
5789  *	@head: list
5790  *
5791  *	This function shuts down a device interface and removes it
5792  *	from the kernel tables.
5793  *	If head not NULL, device is queued to be unregistered later.
5794  *
5795  *	Callers must hold the rtnl semaphore.  You may want
5796  *	unregister_netdev() instead of this.
5797  */
5798 
unregister_netdevice_queue(struct net_device * dev,struct list_head * head)5799 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5800 {
5801 	ASSERT_RTNL();
5802 
5803 	if (head) {
5804 		list_move_tail(&dev->unreg_list, head);
5805 	} else {
5806 		rollback_registered(dev);
5807 		/* Finish processing unregister after unlock */
5808 		net_set_todo(dev);
5809 	}
5810 }
5811 EXPORT_SYMBOL(unregister_netdevice_queue);
5812 
5813 /**
5814  *	unregister_netdevice_many - unregister many devices
5815  *	@head: list of devices
5816  */
unregister_netdevice_many(struct list_head * head)5817 void unregister_netdevice_many(struct list_head *head)
5818 {
5819 	struct net_device *dev;
5820 
5821 	if (!list_empty(head)) {
5822 		rollback_registered_many(head);
5823 		list_for_each_entry(dev, head, unreg_list)
5824 			net_set_todo(dev);
5825 	}
5826 }
5827 EXPORT_SYMBOL(unregister_netdevice_many);
5828 
5829 /**
5830  *	unregister_netdev - remove device from the kernel
5831  *	@dev: device
5832  *
5833  *	This function shuts down a device interface and removes it
5834  *	from the kernel tables.
5835  *
5836  *	This is just a wrapper for unregister_netdevice that takes
5837  *	the rtnl semaphore.  In general you want to use this and not
5838  *	unregister_netdevice.
5839  */
unregister_netdev(struct net_device * dev)5840 void unregister_netdev(struct net_device *dev)
5841 {
5842 	rtnl_lock();
5843 	unregister_netdevice(dev);
5844 	rtnl_unlock();
5845 }
5846 EXPORT_SYMBOL(unregister_netdev);
5847 
5848 /**
5849  *	dev_change_net_namespace - move device to different nethost namespace
5850  *	@dev: device
5851  *	@net: network namespace
5852  *	@pat: If not NULL name pattern to try if the current device name
5853  *	      is already taken in the destination network namespace.
5854  *
5855  *	This function shuts down a device interface and moves it
5856  *	to a new network namespace. On success 0 is returned, on
5857  *	a failure a netagive errno code is returned.
5858  *
5859  *	Callers must hold the rtnl semaphore.
5860  */
5861 
dev_change_net_namespace(struct net_device * dev,struct net * net,const char * pat)5862 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5863 {
5864 	int err;
5865 
5866 	ASSERT_RTNL();
5867 
5868 	/* Don't allow namespace local devices to be moved. */
5869 	err = -EINVAL;
5870 	if (dev->features & NETIF_F_NETNS_LOCAL)
5871 		goto out;
5872 
5873 	/* Ensure the device has been registrered */
5874 	if (dev->reg_state != NETREG_REGISTERED)
5875 		goto out;
5876 
5877 	/* Get out if there is nothing todo */
5878 	err = 0;
5879 	if (net_eq(dev_net(dev), net))
5880 		goto out;
5881 
5882 	/* Pick the destination device name, and ensure
5883 	 * we can use it in the destination network namespace.
5884 	 */
5885 	err = -EEXIST;
5886 	if (__dev_get_by_name(net, dev->name)) {
5887 		/* We get here if we can't use the current device name */
5888 		if (!pat)
5889 			goto out;
5890 		if (dev_get_valid_name(net, dev, pat) < 0)
5891 			goto out;
5892 	}
5893 
5894 	/*
5895 	 * And now a mini version of register_netdevice unregister_netdevice.
5896 	 */
5897 
5898 	/* If device is running close it first. */
5899 	dev_close(dev);
5900 
5901 	/* And unlink it from device chain */
5902 	err = -ENODEV;
5903 	unlist_netdevice(dev);
5904 
5905 	synchronize_net();
5906 
5907 	/* Shutdown queueing discipline. */
5908 	dev_shutdown(dev);
5909 
5910 	/* Notify protocols, that we are about to destroy
5911 	   this device. They should clean all the things.
5912 
5913 	   Note that dev->reg_state stays at NETREG_REGISTERED.
5914 	   This is wanted because this way 8021q and macvlan know
5915 	   the device is just moving and can keep their slaves up.
5916 	*/
5917 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5918 	rcu_barrier();
5919 	call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5920 	rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5921 
5922 	/*
5923 	 *	Flush the unicast and multicast chains
5924 	 */
5925 	dev_uc_flush(dev);
5926 	dev_mc_flush(dev);
5927 
5928 	/* Send a netdev-removed uevent to the old namespace */
5929 	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
5930 
5931 	/* Actually switch the network namespace */
5932 	dev_net_set(dev, net);
5933 
5934 	/* If there is an ifindex conflict assign a new one */
5935 	if (__dev_get_by_index(net, dev->ifindex)) {
5936 		int iflink = (dev->iflink == dev->ifindex);
5937 		dev->ifindex = dev_new_index(net);
5938 		if (iflink)
5939 			dev->iflink = dev->ifindex;
5940 	}
5941 
5942 	/* Send a netdev-add uevent to the new namespace */
5943 	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
5944 
5945 	/* Fixup kobjects */
5946 	err = device_rename(&dev->dev, dev->name);
5947 	WARN_ON(err);
5948 
5949 	/* Add the device back in the hashes */
5950 	list_netdevice(dev);
5951 
5952 	/* Notify protocols, that a new device appeared. */
5953 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
5954 
5955 	/*
5956 	 *	Prevent userspace races by waiting until the network
5957 	 *	device is fully setup before sending notifications.
5958 	 */
5959 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5960 
5961 	synchronize_net();
5962 	err = 0;
5963 out:
5964 	return err;
5965 }
5966 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5967 
dev_cpu_callback(struct notifier_block * nfb,unsigned long action,void * ocpu)5968 static int dev_cpu_callback(struct notifier_block *nfb,
5969 			    unsigned long action,
5970 			    void *ocpu)
5971 {
5972 	struct sk_buff **list_skb;
5973 	struct sk_buff *skb;
5974 	unsigned int cpu, oldcpu = (unsigned long)ocpu;
5975 	struct softnet_data *sd, *oldsd;
5976 
5977 	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
5978 		return NOTIFY_OK;
5979 
5980 	local_irq_disable();
5981 	cpu = smp_processor_id();
5982 	sd = &per_cpu(softnet_data, cpu);
5983 	oldsd = &per_cpu(softnet_data, oldcpu);
5984 
5985 	/* Find end of our completion_queue. */
5986 	list_skb = &sd->completion_queue;
5987 	while (*list_skb)
5988 		list_skb = &(*list_skb)->next;
5989 	/* Append completion queue from offline CPU. */
5990 	*list_skb = oldsd->completion_queue;
5991 	oldsd->completion_queue = NULL;
5992 
5993 	/* Append output queue from offline CPU. */
5994 	if (oldsd->output_queue) {
5995 		*sd->output_queue_tailp = oldsd->output_queue;
5996 		sd->output_queue_tailp = oldsd->output_queue_tailp;
5997 		oldsd->output_queue = NULL;
5998 		oldsd->output_queue_tailp = &oldsd->output_queue;
5999 	}
6000 	/* Append NAPI poll list from offline CPU. */
6001 	if (!list_empty(&oldsd->poll_list)) {
6002 		list_splice_init(&oldsd->poll_list, &sd->poll_list);
6003 		raise_softirq_irqoff(NET_RX_SOFTIRQ);
6004 	}
6005 
6006 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
6007 	local_irq_enable();
6008 
6009 	/* Process offline CPU's input_pkt_queue */
6010 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6011 		netif_rx(skb);
6012 		input_queue_head_incr(oldsd);
6013 	}
6014 	while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
6015 		netif_rx(skb);
6016 		input_queue_head_incr(oldsd);
6017 	}
6018 
6019 	return NOTIFY_OK;
6020 }
6021 
6022 
6023 /**
6024  *	netdev_increment_features - increment feature set by one
6025  *	@all: current feature set
6026  *	@one: new feature set
6027  *	@mask: mask feature set
6028  *
6029  *	Computes a new feature set after adding a device with feature set
6030  *	@one to the master device with current feature set @all.  Will not
6031  *	enable anything that is off in @mask. Returns the new feature set.
6032  */
netdev_increment_features(netdev_features_t all,netdev_features_t one,netdev_features_t mask)6033 netdev_features_t netdev_increment_features(netdev_features_t all,
6034 	netdev_features_t one, netdev_features_t mask)
6035 {
6036 	if (mask & NETIF_F_GEN_CSUM)
6037 		mask |= NETIF_F_ALL_CSUM;
6038 	mask |= NETIF_F_VLAN_CHALLENGED;
6039 
6040 	all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6041 	all &= one | ~NETIF_F_ALL_FOR_ALL;
6042 
6043 	/* If one device supports hw checksumming, set for all. */
6044 	if (all & NETIF_F_GEN_CSUM)
6045 		all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
6046 
6047 	return all;
6048 }
6049 EXPORT_SYMBOL(netdev_increment_features);
6050 
netdev_create_hash(void)6051 static struct hlist_head *netdev_create_hash(void)
6052 {
6053 	int i;
6054 	struct hlist_head *hash;
6055 
6056 	hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6057 	if (hash != NULL)
6058 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
6059 			INIT_HLIST_HEAD(&hash[i]);
6060 
6061 	return hash;
6062 }
6063 
6064 /* Initialize per network namespace state */
netdev_init(struct net * net)6065 static int __net_init netdev_init(struct net *net)
6066 {
6067 	if (net != &init_net)
6068 		INIT_LIST_HEAD(&net->dev_base_head);
6069 
6070 	net->dev_name_head = netdev_create_hash();
6071 	if (net->dev_name_head == NULL)
6072 		goto err_name;
6073 
6074 	net->dev_index_head = netdev_create_hash();
6075 	if (net->dev_index_head == NULL)
6076 		goto err_idx;
6077 
6078 	return 0;
6079 
6080 err_idx:
6081 	kfree(net->dev_name_head);
6082 err_name:
6083 	return -ENOMEM;
6084 }
6085 
6086 /**
6087  *	netdev_drivername - network driver for the device
6088  *	@dev: network device
6089  *
6090  *	Determine network driver for device.
6091  */
netdev_drivername(const struct net_device * dev)6092 const char *netdev_drivername(const struct net_device *dev)
6093 {
6094 	const struct device_driver *driver;
6095 	const struct device *parent;
6096 	const char *empty = "";
6097 
6098 	parent = dev->dev.parent;
6099 	if (!parent)
6100 		return empty;
6101 
6102 	driver = parent->driver;
6103 	if (driver && driver->name)
6104 		return driver->name;
6105 	return empty;
6106 }
6107 
__netdev_printk(const char * level,const struct net_device * dev,struct va_format * vaf)6108 static int __netdev_printk(const char *level, const struct net_device *dev,
6109 			   struct va_format *vaf)
6110 {
6111 	int r;
6112 
6113 	if (dev && dev->dev.parent) {
6114 		r = dev_printk_emit(level[1] - '0',
6115 				    dev->dev.parent,
6116 				    "%s %s %s: %pV",
6117 				    dev_driver_string(dev->dev.parent),
6118 				    dev_name(dev->dev.parent),
6119 				    netdev_name(dev), vaf);
6120 	} else if (dev) {
6121 		r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6122 	} else {
6123 		r = printk("%s(NULL net_device): %pV", level, vaf);
6124 	}
6125 
6126 	return r;
6127 }
6128 
netdev_printk(const char * level,const struct net_device * dev,const char * format,...)6129 int netdev_printk(const char *level, const struct net_device *dev,
6130 		  const char *format, ...)
6131 {
6132 	struct va_format vaf;
6133 	va_list args;
6134 	int r;
6135 
6136 	va_start(args, format);
6137 
6138 	vaf.fmt = format;
6139 	vaf.va = &args;
6140 
6141 	r = __netdev_printk(level, dev, &vaf);
6142 
6143 	va_end(args);
6144 
6145 	return r;
6146 }
6147 EXPORT_SYMBOL(netdev_printk);
6148 
6149 #define define_netdev_printk_level(func, level)			\
6150 int func(const struct net_device *dev, const char *fmt, ...)	\
6151 {								\
6152 	int r;							\
6153 	struct va_format vaf;					\
6154 	va_list args;						\
6155 								\
6156 	va_start(args, fmt);					\
6157 								\
6158 	vaf.fmt = fmt;						\
6159 	vaf.va = &args;						\
6160 								\
6161 	r = __netdev_printk(level, dev, &vaf);			\
6162 								\
6163 	va_end(args);						\
6164 								\
6165 	return r;						\
6166 }								\
6167 EXPORT_SYMBOL(func);
6168 
6169 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6170 define_netdev_printk_level(netdev_alert, KERN_ALERT);
6171 define_netdev_printk_level(netdev_crit, KERN_CRIT);
6172 define_netdev_printk_level(netdev_err, KERN_ERR);
6173 define_netdev_printk_level(netdev_warn, KERN_WARNING);
6174 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6175 define_netdev_printk_level(netdev_info, KERN_INFO);
6176 
netdev_exit(struct net * net)6177 static void __net_exit netdev_exit(struct net *net)
6178 {
6179 	kfree(net->dev_name_head);
6180 	kfree(net->dev_index_head);
6181 }
6182 
6183 static struct pernet_operations __net_initdata netdev_net_ops = {
6184 	.init = netdev_init,
6185 	.exit = netdev_exit,
6186 };
6187 
default_device_exit(struct net * net)6188 static void __net_exit default_device_exit(struct net *net)
6189 {
6190 	struct net_device *dev, *aux;
6191 	/*
6192 	 * Push all migratable network devices back to the
6193 	 * initial network namespace
6194 	 */
6195 	rtnl_lock();
6196 	for_each_netdev_safe(net, dev, aux) {
6197 		int err;
6198 		char fb_name[IFNAMSIZ];
6199 
6200 		/* Ignore unmoveable devices (i.e. loopback) */
6201 		if (dev->features & NETIF_F_NETNS_LOCAL)
6202 			continue;
6203 
6204 		/* Leave virtual devices for the generic cleanup */
6205 		if (dev->rtnl_link_ops)
6206 			continue;
6207 
6208 		/* Push remaining network devices to init_net */
6209 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6210 		err = dev_change_net_namespace(dev, &init_net, fb_name);
6211 		if (err) {
6212 			pr_emerg("%s: failed to move %s to init_net: %d\n",
6213 				 __func__, dev->name, err);
6214 			BUG();
6215 		}
6216 	}
6217 	rtnl_unlock();
6218 }
6219 
default_device_exit_batch(struct list_head * net_list)6220 static void __net_exit default_device_exit_batch(struct list_head *net_list)
6221 {
6222 	/* At exit all network devices most be removed from a network
6223 	 * namespace.  Do this in the reverse order of registration.
6224 	 * Do this across as many network namespaces as possible to
6225 	 * improve batching efficiency.
6226 	 */
6227 	struct net_device *dev;
6228 	struct net *net;
6229 	LIST_HEAD(dev_kill_list);
6230 
6231 	rtnl_lock();
6232 	list_for_each_entry(net, net_list, exit_list) {
6233 		for_each_netdev_reverse(net, dev) {
6234 			if (dev->rtnl_link_ops)
6235 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6236 			else
6237 				unregister_netdevice_queue(dev, &dev_kill_list);
6238 		}
6239 	}
6240 	unregister_netdevice_many(&dev_kill_list);
6241 	list_del(&dev_kill_list);
6242 	rtnl_unlock();
6243 }
6244 
6245 static struct pernet_operations __net_initdata default_device_ops = {
6246 	.exit = default_device_exit,
6247 	.exit_batch = default_device_exit_batch,
6248 };
6249 
6250 /*
6251  *	Initialize the DEV module. At boot time this walks the device list and
6252  *	unhooks any devices that fail to initialise (normally hardware not
6253  *	present) and leaves us with a valid list of present and active devices.
6254  *
6255  */
6256 
6257 /*
6258  *       This is called single threaded during boot, so no need
6259  *       to take the rtnl semaphore.
6260  */
net_dev_init(void)6261 static int __init net_dev_init(void)
6262 {
6263 	int i, rc = -ENOMEM;
6264 
6265 	BUG_ON(!dev_boot_phase);
6266 
6267 	if (dev_proc_init())
6268 		goto out;
6269 
6270 	if (netdev_kobject_init())
6271 		goto out;
6272 
6273 	INIT_LIST_HEAD(&ptype_all);
6274 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
6275 		INIT_LIST_HEAD(&ptype_base[i]);
6276 
6277 	INIT_LIST_HEAD(&offload_base);
6278 
6279 	if (register_pernet_subsys(&netdev_net_ops))
6280 		goto out;
6281 
6282 	/*
6283 	 *	Initialise the packet receive queues.
6284 	 */
6285 
6286 	for_each_possible_cpu(i) {
6287 		struct softnet_data *sd = &per_cpu(softnet_data, i);
6288 
6289 		memset(sd, 0, sizeof(*sd));
6290 		skb_queue_head_init(&sd->input_pkt_queue);
6291 		skb_queue_head_init(&sd->process_queue);
6292 		sd->completion_queue = NULL;
6293 		INIT_LIST_HEAD(&sd->poll_list);
6294 		sd->output_queue = NULL;
6295 		sd->output_queue_tailp = &sd->output_queue;
6296 #ifdef CONFIG_RPS
6297 		sd->csd.func = rps_trigger_softirq;
6298 		sd->csd.info = sd;
6299 		sd->csd.flags = 0;
6300 		sd->cpu = i;
6301 #endif
6302 
6303 		sd->backlog.poll = process_backlog;
6304 		sd->backlog.weight = weight_p;
6305 		sd->backlog.gro_list = NULL;
6306 		sd->backlog.gro_count = 0;
6307 	}
6308 
6309 	dev_boot_phase = 0;
6310 
6311 	/* The loopback device is special if any other network devices
6312 	 * is present in a network namespace the loopback device must
6313 	 * be present. Since we now dynamically allocate and free the
6314 	 * loopback device ensure this invariant is maintained by
6315 	 * keeping the loopback device as the first device on the
6316 	 * list of network devices.  Ensuring the loopback devices
6317 	 * is the first device that appears and the last network device
6318 	 * that disappears.
6319 	 */
6320 	if (register_pernet_device(&loopback_net_ops))
6321 		goto out;
6322 
6323 	if (register_pernet_device(&default_device_ops))
6324 		goto out;
6325 
6326 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6327 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
6328 
6329 	hotcpu_notifier(dev_cpu_callback, 0);
6330 	dst_init();
6331 	rc = 0;
6332 out:
6333 	return rc;
6334 }
6335 
6336 subsys_initcall(net_dev_init);
6337