• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *      NET3    Protocol independent device support routines.
3  *
4  *		This program is free software; you can redistribute it and/or
5  *		modify it under the terms of the GNU General Public License
6  *		as published by the Free Software Foundation; either version
7  *		2 of the License, or (at your option) any later version.
8  *
9  *	Derived from the non IP parts of dev.c 1.0.19
10  *              Authors:	Ross Biro
11  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *
14  *	Additional Authors:
15  *		Florian la Roche <rzsfl@rz.uni-sb.de>
16  *		Alan Cox <gw4pts@gw4pts.ampr.org>
17  *		David Hinds <dahinds@users.sourceforge.net>
18  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19  *		Adam Sulmicki <adam@cfar.umd.edu>
20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
21  *
22  *	Changes:
23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
24  *                                      to 2 if register_netdev gets called
25  *                                      before net_dev_init & also removed a
26  *                                      few lines of code in the process.
27  *		Alan Cox	:	device private ioctl copies fields back.
28  *		Alan Cox	:	Transmit queue code does relevant
29  *					stunts to keep the queue safe.
30  *		Alan Cox	:	Fixed double lock.
31  *		Alan Cox	:	Fixed promisc NULL pointer trap
32  *		????????	:	Support the full private ioctl range
33  *		Alan Cox	:	Moved ioctl permission check into
34  *					drivers
35  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
36  *		Alan Cox	:	100 backlog just doesn't cut it when
37  *					you start doing multicast video 8)
38  *		Alan Cox	:	Rewrote net_bh and list manager.
39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
40  *		Alan Cox	:	Took out transmit every packet pass
41  *					Saved a few bytes in the ioctl handler
42  *		Alan Cox	:	Network driver sets packet type before
43  *					calling netif_rx. Saves a function
44  *					call a packet.
45  *		Alan Cox	:	Hashed net_bh()
46  *		Richard Kooijman:	Timestamp fixes.
47  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
48  *		Alan Cox	:	Device lock protection.
49  *              Alan Cox        :       Fixed nasty side effect of device close
50  *					changes.
51  *		Rudi Cilibrasi	:	Pass the right thing to
52  *					set_mac_address()
53  *		Dave Miller	:	32bit quantity for the device lock to
54  *					make it work out on a Sparc.
55  *		Bjorn Ekwall	:	Added KERNELD hack.
56  *		Alan Cox	:	Cleaned up the backlog initialise.
57  *		Craig Metz	:	SIOCGIFCONF fix if space for under
58  *					1 device.
59  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
60  *					is no device open function.
61  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
62  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
63  *		Cyrus Durgin	:	Cleaned for KMOD
64  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
65  *					A network device unload needs to purge
66  *					the backlog queue.
67  *	Paul Rusty Russell	:	SIOCSIFNAME
68  *              Pekka Riikonen  :	Netdev boot-time settings code
69  *              Andrew Morton   :       Make unregister_netdevice wait
70  *                                      indefinitely on dev->refcnt
71  *              J Hadi Salim    :       - Backlog queue sampling
72  *				        - netif_rx() feedback
73  */
74 
75 #include <linux/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/sched/mm.h>
85 #include <linux/mutex.h>
86 #include <linux/string.h>
87 #include <linux/mm.h>
88 #include <linux/socket.h>
89 #include <linux/sockios.h>
90 #include <linux/errno.h>
91 #include <linux/interrupt.h>
92 #include <linux/if_ether.h>
93 #include <linux/netdevice.h>
94 #include <linux/etherdevice.h>
95 #include <linux/ethtool.h>
96 #include <linux/notifier.h>
97 #include <linux/skbuff.h>
98 #include <linux/bpf.h>
99 #include <linux/bpf_trace.h>
100 #include <net/net_namespace.h>
101 #include <net/sock.h>
102 #include <net/busy_poll.h>
103 #include <linux/rtnetlink.h>
104 #include <linux/stat.h>
105 #include <net/dst.h>
106 #include <net/dst_metadata.h>
107 #include <net/pkt_sched.h>
108 #include <net/pkt_cls.h>
109 #include <net/checksum.h>
110 #include <net/xfrm.h>
111 #include <linux/highmem.h>
112 #include <linux/init.h>
113 #include <linux/module.h>
114 #include <linux/netpoll.h>
115 #include <linux/rcupdate.h>
116 #include <linux/delay.h>
117 #include <net/iw_handler.h>
118 #include <asm/current.h>
119 #include <linux/audit.h>
120 #include <linux/dmaengine.h>
121 #include <linux/err.h>
122 #include <linux/ctype.h>
123 #include <linux/if_arp.h>
124 #include <linux/if_vlan.h>
125 #include <linux/ip.h>
126 #include <net/ip.h>
127 #include <net/mpls.h>
128 #include <linux/ipv6.h>
129 #include <linux/in.h>
130 #include <linux/jhash.h>
131 #include <linux/random.h>
132 #include <trace/events/napi.h>
133 #include <trace/events/net.h>
134 #include <trace/events/skb.h>
135 #include <linux/pci.h>
136 #include <linux/inetdevice.h>
137 #include <linux/cpu_rmap.h>
138 #include <linux/static_key.h>
139 #include <linux/hashtable.h>
140 #include <linux/vmalloc.h>
141 #include <linux/if_macvlan.h>
142 #include <linux/errqueue.h>
143 #include <linux/hrtimer.h>
144 #include <linux/netfilter_ingress.h>
145 #include <linux/crash_dump.h>
146 #include <linux/sctp.h>
147 #include <net/udp_tunnel.h>
148 
149 #include "net-sysfs.h"
150 
151 /* Instead of increasing this, you should create a hash table. */
152 #define MAX_GRO_SKBS 8
153 
154 /* This should be increased if a protocol with a bigger head is added. */
155 #define GRO_MAX_HEAD (MAX_HEADER + 128)
156 
157 static DEFINE_SPINLOCK(ptype_lock);
158 static DEFINE_SPINLOCK(offload_lock);
159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160 struct list_head ptype_all __read_mostly;	/* Taps */
161 static struct list_head offload_base __read_mostly;
162 
163 static int netif_rx_internal(struct sk_buff *skb);
164 static int call_netdevice_notifiers_info(unsigned long val,
165 					 struct net_device *dev,
166 					 struct netdev_notifier_info *info);
167 static struct napi_struct *napi_by_id(unsigned int napi_id);
168 
169 /*
170  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
171  * semaphore.
172  *
173  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
174  *
175  * Writers must hold the rtnl semaphore while they loop through the
176  * dev_base_head list, and hold dev_base_lock for writing when they do the
177  * actual updates.  This allows pure readers to access the list even
178  * while a writer is preparing to update it.
179  *
180  * To put it another way, dev_base_lock is held for writing only to
181  * protect against pure readers; the rtnl semaphore provides the
182  * protection against other writers.
183  *
184  * See, for example usages, register_netdevice() and
185  * unregister_netdevice(), which must be called with the rtnl
186  * semaphore held.
187  */
188 DEFINE_RWLOCK(dev_base_lock);
189 EXPORT_SYMBOL(dev_base_lock);
190 
191 /* protects napi_hash addition/deletion and napi_gen_id */
192 static DEFINE_SPINLOCK(napi_hash_lock);
193 
194 static unsigned int napi_gen_id = NR_CPUS;
195 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
196 
197 static seqcount_t devnet_rename_seq;
198 
dev_base_seq_inc(struct net * net)199 static inline void dev_base_seq_inc(struct net *net)
200 {
201 	while (++net->dev_base_seq == 0)
202 		;
203 }
204 
dev_name_hash(struct net * net,const char * name)205 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
206 {
207 	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
208 
209 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
210 }
211 
dev_index_hash(struct net * net,int ifindex)212 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
213 {
214 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
215 }
216 
rps_lock(struct softnet_data * sd)217 static inline void rps_lock(struct softnet_data *sd)
218 {
219 #ifdef CONFIG_RPS
220 	spin_lock(&sd->input_pkt_queue.lock);
221 #endif
222 }
223 
rps_unlock(struct softnet_data * sd)224 static inline void rps_unlock(struct softnet_data *sd)
225 {
226 #ifdef CONFIG_RPS
227 	spin_unlock(&sd->input_pkt_queue.lock);
228 #endif
229 }
230 
231 /* Device list insertion */
list_netdevice(struct net_device * dev)232 static void list_netdevice(struct net_device *dev)
233 {
234 	struct net *net = dev_net(dev);
235 
236 	ASSERT_RTNL();
237 
238 	write_lock_bh(&dev_base_lock);
239 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
240 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
241 	hlist_add_head_rcu(&dev->index_hlist,
242 			   dev_index_hash(net, dev->ifindex));
243 	write_unlock_bh(&dev_base_lock);
244 
245 	dev_base_seq_inc(net);
246 }
247 
248 /* Device list removal
249  * caller must respect a RCU grace period before freeing/reusing dev
250  */
unlist_netdevice(struct net_device * dev)251 static void unlist_netdevice(struct net_device *dev)
252 {
253 	ASSERT_RTNL();
254 
255 	/* Unlink dev from the device chain */
256 	write_lock_bh(&dev_base_lock);
257 	list_del_rcu(&dev->dev_list);
258 	hlist_del_rcu(&dev->name_hlist);
259 	hlist_del_rcu(&dev->index_hlist);
260 	write_unlock_bh(&dev_base_lock);
261 
262 	dev_base_seq_inc(dev_net(dev));
263 }
264 
265 /*
266  *	Our notifier list
267  */
268 
269 static RAW_NOTIFIER_HEAD(netdev_chain);
270 
271 /*
272  *	Device drivers call our routines to queue packets here. We empty the
273  *	queue in the local softnet handler.
274  */
275 
276 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
277 EXPORT_PER_CPU_SYMBOL(softnet_data);
278 
279 #ifdef CONFIG_LOCKDEP
280 /*
281  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
282  * according to dev->type
283  */
284 static const unsigned short netdev_lock_type[] = {
285 	 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
286 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
287 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
288 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
289 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
290 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
291 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
292 	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
293 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
294 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
295 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
296 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
297 	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
298 	 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
299 	 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
300 
301 static const char *const netdev_lock_name[] = {
302 	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
303 	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
304 	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
305 	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
306 	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
307 	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
308 	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
309 	"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
310 	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
311 	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
312 	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
313 	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
314 	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
315 	"_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
316 	"_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
317 
318 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
319 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
320 
netdev_lock_pos(unsigned short dev_type)321 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
322 {
323 	int i;
324 
325 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
326 		if (netdev_lock_type[i] == dev_type)
327 			return i;
328 	/* the last key is used by default */
329 	return ARRAY_SIZE(netdev_lock_type) - 1;
330 }
331 
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)332 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
333 						 unsigned short dev_type)
334 {
335 	int i;
336 
337 	i = netdev_lock_pos(dev_type);
338 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
339 				   netdev_lock_name[i]);
340 }
341 
netdev_set_addr_lockdep_class(struct net_device * dev)342 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
343 {
344 	int i;
345 
346 	i = netdev_lock_pos(dev->type);
347 	lockdep_set_class_and_name(&dev->addr_list_lock,
348 				   &netdev_addr_lock_key[i],
349 				   netdev_lock_name[i]);
350 }
351 #else
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)352 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
353 						 unsigned short dev_type)
354 {
355 }
netdev_set_addr_lockdep_class(struct net_device * dev)356 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
357 {
358 }
359 #endif
360 
361 /*******************************************************************************
362  *
363  *		Protocol management and registration routines
364  *
365  *******************************************************************************/
366 
367 
368 /*
369  *	Add a protocol ID to the list. Now that the input handler is
370  *	smarter we can dispense with all the messy stuff that used to be
371  *	here.
372  *
373  *	BEWARE!!! Protocol handlers, mangling input packets,
374  *	MUST BE last in hash buckets and checking protocol handlers
375  *	MUST start from promiscuous ptype_all chain in net_bh.
376  *	It is true now, do not change it.
377  *	Explanation follows: if protocol handler, mangling packet, will
378  *	be the first on list, it is not able to sense, that packet
379  *	is cloned and should be copied-on-write, so that it will
380  *	change it and subsequent readers will get broken packet.
381  *							--ANK (980803)
382  */
383 
ptype_head(const struct packet_type * pt)384 static inline struct list_head *ptype_head(const struct packet_type *pt)
385 {
386 	if (pt->type == htons(ETH_P_ALL))
387 		return pt->dev ? &pt->dev->ptype_all : &ptype_all;
388 	else
389 		return pt->dev ? &pt->dev->ptype_specific :
390 				 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
391 }
392 
393 /**
394  *	dev_add_pack - add packet handler
395  *	@pt: packet type declaration
396  *
397  *	Add a protocol handler to the networking stack. The passed &packet_type
398  *	is linked into kernel lists and may not be freed until it has been
399  *	removed from the kernel lists.
400  *
401  *	This call does not sleep therefore it can not
402  *	guarantee all CPU's that are in middle of receiving packets
403  *	will see the new packet type (until the next received packet).
404  */
405 
dev_add_pack(struct packet_type * pt)406 void dev_add_pack(struct packet_type *pt)
407 {
408 	struct list_head *head = ptype_head(pt);
409 
410 	spin_lock(&ptype_lock);
411 	list_add_rcu(&pt->list, head);
412 	spin_unlock(&ptype_lock);
413 }
414 EXPORT_SYMBOL(dev_add_pack);
415 
416 /**
417  *	__dev_remove_pack	 - remove packet handler
418  *	@pt: packet type declaration
419  *
420  *	Remove a protocol handler that was previously added to the kernel
421  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
422  *	from the kernel lists and can be freed or reused once this function
423  *	returns.
424  *
425  *      The packet type might still be in use by receivers
426  *	and must not be freed until after all the CPU's have gone
427  *	through a quiescent state.
428  */
__dev_remove_pack(struct packet_type * pt)429 void __dev_remove_pack(struct packet_type *pt)
430 {
431 	struct list_head *head = ptype_head(pt);
432 	struct packet_type *pt1;
433 
434 	spin_lock(&ptype_lock);
435 
436 	list_for_each_entry(pt1, head, list) {
437 		if (pt == pt1) {
438 			list_del_rcu(&pt->list);
439 			goto out;
440 		}
441 	}
442 
443 	pr_warn("dev_remove_pack: %p not found\n", pt);
444 out:
445 	spin_unlock(&ptype_lock);
446 }
447 EXPORT_SYMBOL(__dev_remove_pack);
448 
449 /**
450  *	dev_remove_pack	 - remove packet handler
451  *	@pt: packet type declaration
452  *
453  *	Remove a protocol handler that was previously added to the kernel
454  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
455  *	from the kernel lists and can be freed or reused once this function
456  *	returns.
457  *
458  *	This call sleeps to guarantee that no CPU is looking at the packet
459  *	type after return.
460  */
dev_remove_pack(struct packet_type * pt)461 void dev_remove_pack(struct packet_type *pt)
462 {
463 	__dev_remove_pack(pt);
464 
465 	synchronize_net();
466 }
467 EXPORT_SYMBOL(dev_remove_pack);
468 
469 
470 /**
471  *	dev_add_offload - register offload handlers
472  *	@po: protocol offload declaration
473  *
474  *	Add protocol offload handlers to the networking stack. The passed
475  *	&proto_offload is linked into kernel lists and may not be freed until
476  *	it has been removed from the kernel lists.
477  *
478  *	This call does not sleep therefore it can not
479  *	guarantee all CPU's that are in middle of receiving packets
480  *	will see the new offload handlers (until the next received packet).
481  */
dev_add_offload(struct packet_offload * po)482 void dev_add_offload(struct packet_offload *po)
483 {
484 	struct packet_offload *elem;
485 
486 	spin_lock(&offload_lock);
487 	list_for_each_entry(elem, &offload_base, list) {
488 		if (po->priority < elem->priority)
489 			break;
490 	}
491 	list_add_rcu(&po->list, elem->list.prev);
492 	spin_unlock(&offload_lock);
493 }
494 EXPORT_SYMBOL(dev_add_offload);
495 
496 /**
497  *	__dev_remove_offload	 - remove offload handler
498  *	@po: packet offload declaration
499  *
500  *	Remove a protocol offload handler that was previously added to the
501  *	kernel offload handlers by dev_add_offload(). The passed &offload_type
502  *	is removed from the kernel lists and can be freed or reused once this
503  *	function returns.
504  *
505  *      The packet type might still be in use by receivers
506  *	and must not be freed until after all the CPU's have gone
507  *	through a quiescent state.
508  */
__dev_remove_offload(struct packet_offload * po)509 static void __dev_remove_offload(struct packet_offload *po)
510 {
511 	struct list_head *head = &offload_base;
512 	struct packet_offload *po1;
513 
514 	spin_lock(&offload_lock);
515 
516 	list_for_each_entry(po1, head, list) {
517 		if (po == po1) {
518 			list_del_rcu(&po->list);
519 			goto out;
520 		}
521 	}
522 
523 	pr_warn("dev_remove_offload: %p not found\n", po);
524 out:
525 	spin_unlock(&offload_lock);
526 }
527 
528 /**
529  *	dev_remove_offload	 - remove packet offload handler
530  *	@po: packet offload declaration
531  *
532  *	Remove a packet offload handler that was previously added to the kernel
533  *	offload handlers by dev_add_offload(). The passed &offload_type is
534  *	removed from the kernel lists and can be freed or reused once this
535  *	function returns.
536  *
537  *	This call sleeps to guarantee that no CPU is looking at the packet
538  *	type after return.
539  */
dev_remove_offload(struct packet_offload * po)540 void dev_remove_offload(struct packet_offload *po)
541 {
542 	__dev_remove_offload(po);
543 
544 	synchronize_net();
545 }
546 EXPORT_SYMBOL(dev_remove_offload);
547 
548 /******************************************************************************
549  *
550  *		      Device Boot-time Settings Routines
551  *
552  ******************************************************************************/
553 
554 /* Boot time configuration table */
555 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
556 
557 /**
558  *	netdev_boot_setup_add	- add new setup entry
559  *	@name: name of the device
560  *	@map: configured settings for the device
561  *
562  *	Adds new setup entry to the dev_boot_setup list.  The function
563  *	returns 0 on error and 1 on success.  This is a generic routine to
564  *	all netdevices.
565  */
netdev_boot_setup_add(char * name,struct ifmap * map)566 static int netdev_boot_setup_add(char *name, struct ifmap *map)
567 {
568 	struct netdev_boot_setup *s;
569 	int i;
570 
571 	s = dev_boot_setup;
572 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
573 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
574 			memset(s[i].name, 0, sizeof(s[i].name));
575 			strlcpy(s[i].name, name, IFNAMSIZ);
576 			memcpy(&s[i].map, map, sizeof(s[i].map));
577 			break;
578 		}
579 	}
580 
581 	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
582 }
583 
584 /**
585  * netdev_boot_setup_check	- check boot time settings
586  * @dev: the netdevice
587  *
588  * Check boot time settings for the device.
589  * The found settings are set for the device to be used
590  * later in the device probing.
591  * Returns 0 if no settings found, 1 if they are.
592  */
netdev_boot_setup_check(struct net_device * dev)593 int netdev_boot_setup_check(struct net_device *dev)
594 {
595 	struct netdev_boot_setup *s = dev_boot_setup;
596 	int i;
597 
598 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
599 		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
600 		    !strcmp(dev->name, s[i].name)) {
601 			dev->irq = s[i].map.irq;
602 			dev->base_addr = s[i].map.base_addr;
603 			dev->mem_start = s[i].map.mem_start;
604 			dev->mem_end = s[i].map.mem_end;
605 			return 1;
606 		}
607 	}
608 	return 0;
609 }
610 EXPORT_SYMBOL(netdev_boot_setup_check);
611 
612 
613 /**
614  * netdev_boot_base	- get address from boot time settings
615  * @prefix: prefix for network device
616  * @unit: id for network device
617  *
618  * Check boot time settings for the base address of device.
619  * The found settings are set for the device to be used
620  * later in the device probing.
621  * Returns 0 if no settings found.
622  */
netdev_boot_base(const char * prefix,int unit)623 unsigned long netdev_boot_base(const char *prefix, int unit)
624 {
625 	const struct netdev_boot_setup *s = dev_boot_setup;
626 	char name[IFNAMSIZ];
627 	int i;
628 
629 	sprintf(name, "%s%d", prefix, unit);
630 
631 	/*
632 	 * If device already registered then return base of 1
633 	 * to indicate not to probe for this interface
634 	 */
635 	if (__dev_get_by_name(&init_net, name))
636 		return 1;
637 
638 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
639 		if (!strcmp(name, s[i].name))
640 			return s[i].map.base_addr;
641 	return 0;
642 }
643 
644 /*
645  * Saves at boot time configured settings for any netdevice.
646  */
netdev_boot_setup(char * str)647 int __init netdev_boot_setup(char *str)
648 {
649 	int ints[5];
650 	struct ifmap map;
651 
652 	str = get_options(str, ARRAY_SIZE(ints), ints);
653 	if (!str || !*str)
654 		return 0;
655 
656 	/* Save settings */
657 	memset(&map, 0, sizeof(map));
658 	if (ints[0] > 0)
659 		map.irq = ints[1];
660 	if (ints[0] > 1)
661 		map.base_addr = ints[2];
662 	if (ints[0] > 2)
663 		map.mem_start = ints[3];
664 	if (ints[0] > 3)
665 		map.mem_end = ints[4];
666 
667 	/* Add new entry to the list */
668 	return netdev_boot_setup_add(str, &map);
669 }
670 
671 __setup("netdev=", netdev_boot_setup);
672 
673 /*******************************************************************************
674  *
675  *			    Device Interface Subroutines
676  *
677  *******************************************************************************/
678 
679 /**
680  *	dev_get_iflink	- get 'iflink' value of a interface
681  *	@dev: targeted interface
682  *
683  *	Indicates the ifindex the interface is linked to.
684  *	Physical interfaces have the same 'ifindex' and 'iflink' values.
685  */
686 
dev_get_iflink(const struct net_device * dev)687 int dev_get_iflink(const struct net_device *dev)
688 {
689 	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
690 		return dev->netdev_ops->ndo_get_iflink(dev);
691 
692 	return dev->ifindex;
693 }
694 EXPORT_SYMBOL(dev_get_iflink);
695 
696 /**
697  *	dev_fill_metadata_dst - Retrieve tunnel egress information.
698  *	@dev: targeted interface
699  *	@skb: The packet.
700  *
701  *	For better visibility of tunnel traffic OVS needs to retrieve
702  *	egress tunnel information for a packet. Following API allows
703  *	user to get this info.
704  */
dev_fill_metadata_dst(struct net_device * dev,struct sk_buff * skb)705 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
706 {
707 	struct ip_tunnel_info *info;
708 
709 	if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
710 		return -EINVAL;
711 
712 	info = skb_tunnel_info_unclone(skb);
713 	if (!info)
714 		return -ENOMEM;
715 	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
716 		return -EINVAL;
717 
718 	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
719 }
720 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
721 
722 /**
723  *	__dev_get_by_name	- find a device by its name
724  *	@net: the applicable net namespace
725  *	@name: name to find
726  *
727  *	Find an interface by name. Must be called under RTNL semaphore
728  *	or @dev_base_lock. If the name is found a pointer to the device
729  *	is returned. If the name is not found then %NULL is returned. The
730  *	reference counters are not incremented so the caller must be
731  *	careful with locks.
732  */
733 
__dev_get_by_name(struct net * net,const char * name)734 struct net_device *__dev_get_by_name(struct net *net, const char *name)
735 {
736 	struct net_device *dev;
737 	struct hlist_head *head = dev_name_hash(net, name);
738 
739 	hlist_for_each_entry(dev, head, name_hlist)
740 		if (!strncmp(dev->name, name, IFNAMSIZ))
741 			return dev;
742 
743 	return NULL;
744 }
745 EXPORT_SYMBOL(__dev_get_by_name);
746 
747 /**
748  * dev_get_by_name_rcu	- find a device by its name
749  * @net: the applicable net namespace
750  * @name: name to find
751  *
752  * Find an interface by name.
753  * If the name is found a pointer to the device is returned.
754  * If the name is not found then %NULL is returned.
755  * The reference counters are not incremented so the caller must be
756  * careful with locks. The caller must hold RCU lock.
757  */
758 
dev_get_by_name_rcu(struct net * net,const char * name)759 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
760 {
761 	struct net_device *dev;
762 	struct hlist_head *head = dev_name_hash(net, name);
763 
764 	hlist_for_each_entry_rcu(dev, head, name_hlist)
765 		if (!strncmp(dev->name, name, IFNAMSIZ))
766 			return dev;
767 
768 	return NULL;
769 }
770 EXPORT_SYMBOL(dev_get_by_name_rcu);
771 
772 /**
773  *	dev_get_by_name		- find a device by its name
774  *	@net: the applicable net namespace
775  *	@name: name to find
776  *
777  *	Find an interface by name. This can be called from any
778  *	context and does its own locking. The returned handle has
779  *	the usage count incremented and the caller must use dev_put() to
780  *	release it when it is no longer needed. %NULL is returned if no
781  *	matching device is found.
782  */
783 
dev_get_by_name(struct net * net,const char * name)784 struct net_device *dev_get_by_name(struct net *net, const char *name)
785 {
786 	struct net_device *dev;
787 
788 	rcu_read_lock();
789 	dev = dev_get_by_name_rcu(net, name);
790 	if (dev)
791 		dev_hold(dev);
792 	rcu_read_unlock();
793 	return dev;
794 }
795 EXPORT_SYMBOL(dev_get_by_name);
796 
797 /**
798  *	__dev_get_by_index - find a device by its ifindex
799  *	@net: the applicable net namespace
800  *	@ifindex: index of device
801  *
802  *	Search for an interface by index. Returns %NULL if the device
803  *	is not found or a pointer to the device. The device has not
804  *	had its reference counter increased so the caller must be careful
805  *	about locking. The caller must hold either the RTNL semaphore
806  *	or @dev_base_lock.
807  */
808 
__dev_get_by_index(struct net * net,int ifindex)809 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
810 {
811 	struct net_device *dev;
812 	struct hlist_head *head = dev_index_hash(net, ifindex);
813 
814 	hlist_for_each_entry(dev, head, index_hlist)
815 		if (dev->ifindex == ifindex)
816 			return dev;
817 
818 	return NULL;
819 }
820 EXPORT_SYMBOL(__dev_get_by_index);
821 
822 /**
823  *	dev_get_by_index_rcu - find a device by its ifindex
824  *	@net: the applicable net namespace
825  *	@ifindex: index of device
826  *
827  *	Search for an interface by index. Returns %NULL if the device
828  *	is not found or a pointer to the device. The device has not
829  *	had its reference counter increased so the caller must be careful
830  *	about locking. The caller must hold RCU lock.
831  */
832 
dev_get_by_index_rcu(struct net * net,int ifindex)833 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
834 {
835 	struct net_device *dev;
836 	struct hlist_head *head = dev_index_hash(net, ifindex);
837 
838 	hlist_for_each_entry_rcu(dev, head, index_hlist)
839 		if (dev->ifindex == ifindex)
840 			return dev;
841 
842 	return NULL;
843 }
844 EXPORT_SYMBOL(dev_get_by_index_rcu);
845 
846 
847 /**
848  *	dev_get_by_index - find a device by its ifindex
849  *	@net: the applicable net namespace
850  *	@ifindex: index of device
851  *
852  *	Search for an interface by index. Returns NULL if the device
853  *	is not found or a pointer to the device. The device returned has
854  *	had a reference added and the pointer is safe until the user calls
855  *	dev_put to indicate they have finished with it.
856  */
857 
dev_get_by_index(struct net * net,int ifindex)858 struct net_device *dev_get_by_index(struct net *net, int ifindex)
859 {
860 	struct net_device *dev;
861 
862 	rcu_read_lock();
863 	dev = dev_get_by_index_rcu(net, ifindex);
864 	if (dev)
865 		dev_hold(dev);
866 	rcu_read_unlock();
867 	return dev;
868 }
869 EXPORT_SYMBOL(dev_get_by_index);
870 
871 /**
872  *	dev_get_by_napi_id - find a device by napi_id
873  *	@napi_id: ID of the NAPI struct
874  *
875  *	Search for an interface by NAPI ID. Returns %NULL if the device
876  *	is not found or a pointer to the device. The device has not had
877  *	its reference counter increased so the caller must be careful
878  *	about locking. The caller must hold RCU lock.
879  */
880 
dev_get_by_napi_id(unsigned int napi_id)881 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
882 {
883 	struct napi_struct *napi;
884 
885 	WARN_ON_ONCE(!rcu_read_lock_held());
886 
887 	if (napi_id < MIN_NAPI_ID)
888 		return NULL;
889 
890 	napi = napi_by_id(napi_id);
891 
892 	return napi ? napi->dev : NULL;
893 }
894 EXPORT_SYMBOL(dev_get_by_napi_id);
895 
896 /**
897  *	netdev_get_name - get a netdevice name, knowing its ifindex.
898  *	@net: network namespace
899  *	@name: a pointer to the buffer where the name will be stored.
900  *	@ifindex: the ifindex of the interface to get the name from.
901  *
902  *	The use of raw_seqcount_begin() and cond_resched() before
903  *	retrying is required as we want to give the writers a chance
904  *	to complete when CONFIG_PREEMPT is not set.
905  */
netdev_get_name(struct net * net,char * name,int ifindex)906 int netdev_get_name(struct net *net, char *name, int ifindex)
907 {
908 	struct net_device *dev;
909 	unsigned int seq;
910 
911 retry:
912 	seq = raw_seqcount_begin(&devnet_rename_seq);
913 	rcu_read_lock();
914 	dev = dev_get_by_index_rcu(net, ifindex);
915 	if (!dev) {
916 		rcu_read_unlock();
917 		return -ENODEV;
918 	}
919 
920 	strcpy(name, dev->name);
921 	rcu_read_unlock();
922 	if (read_seqcount_retry(&devnet_rename_seq, seq)) {
923 		cond_resched();
924 		goto retry;
925 	}
926 
927 	return 0;
928 }
929 
930 /**
931  *	dev_getbyhwaddr_rcu - find a device by its hardware address
932  *	@net: the applicable net namespace
933  *	@type: media type of device
934  *	@ha: hardware address
935  *
936  *	Search for an interface by MAC address. Returns NULL if the device
937  *	is not found or a pointer to the device.
938  *	The caller must hold RCU or RTNL.
939  *	The returned device has not had its ref count increased
940  *	and the caller must therefore be careful about locking
941  *
942  */
943 
dev_getbyhwaddr_rcu(struct net * net,unsigned short type,const char * ha)944 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
945 				       const char *ha)
946 {
947 	struct net_device *dev;
948 
949 	for_each_netdev_rcu(net, dev)
950 		if (dev->type == type &&
951 		    !memcmp(dev->dev_addr, ha, dev->addr_len))
952 			return dev;
953 
954 	return NULL;
955 }
956 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
957 
__dev_getfirstbyhwtype(struct net * net,unsigned short type)958 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
959 {
960 	struct net_device *dev;
961 
962 	ASSERT_RTNL();
963 	for_each_netdev(net, dev)
964 		if (dev->type == type)
965 			return dev;
966 
967 	return NULL;
968 }
969 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
970 
dev_getfirstbyhwtype(struct net * net,unsigned short type)971 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
972 {
973 	struct net_device *dev, *ret = NULL;
974 
975 	rcu_read_lock();
976 	for_each_netdev_rcu(net, dev)
977 		if (dev->type == type) {
978 			dev_hold(dev);
979 			ret = dev;
980 			break;
981 		}
982 	rcu_read_unlock();
983 	return ret;
984 }
985 EXPORT_SYMBOL(dev_getfirstbyhwtype);
986 
987 /**
988  *	__dev_get_by_flags - find any device with given flags
989  *	@net: the applicable net namespace
990  *	@if_flags: IFF_* values
991  *	@mask: bitmask of bits in if_flags to check
992  *
993  *	Search for any interface with the given flags. Returns NULL if a device
994  *	is not found or a pointer to the device. Must be called inside
995  *	rtnl_lock(), and result refcount is unchanged.
996  */
997 
__dev_get_by_flags(struct net * net,unsigned short if_flags,unsigned short mask)998 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
999 				      unsigned short mask)
1000 {
1001 	struct net_device *dev, *ret;
1002 
1003 	ASSERT_RTNL();
1004 
1005 	ret = NULL;
1006 	for_each_netdev(net, dev) {
1007 		if (((dev->flags ^ if_flags) & mask) == 0) {
1008 			ret = dev;
1009 			break;
1010 		}
1011 	}
1012 	return ret;
1013 }
1014 EXPORT_SYMBOL(__dev_get_by_flags);
1015 
1016 /**
1017  *	dev_valid_name - check if name is okay for network device
1018  *	@name: name string
1019  *
1020  *	Network device names need to be valid file names to
1021  *	to allow sysfs to work.  We also disallow any kind of
1022  *	whitespace.
1023  */
dev_valid_name(const char * name)1024 bool dev_valid_name(const char *name)
1025 {
1026 	if (*name == '\0')
1027 		return false;
1028 	if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1029 		return false;
1030 	if (!strcmp(name, ".") || !strcmp(name, ".."))
1031 		return false;
1032 
1033 	while (*name) {
1034 		if (*name == '/' || *name == ':' || isspace(*name))
1035 			return false;
1036 		name++;
1037 	}
1038 	return true;
1039 }
1040 EXPORT_SYMBOL(dev_valid_name);
1041 
1042 /**
1043  *	__dev_alloc_name - allocate a name for a device
1044  *	@net: network namespace to allocate the device name in
1045  *	@name: name format string
1046  *	@buf:  scratch buffer and result name string
1047  *
1048  *	Passed a format string - eg "lt%d" it will try and find a suitable
1049  *	id. It scans list of devices to build up a free map, then chooses
1050  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1051  *	while allocating the name and adding the device in order to avoid
1052  *	duplicates.
1053  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1054  *	Returns the number of the unit assigned or a negative errno code.
1055  */
1056 
__dev_alloc_name(struct net * net,const char * name,char * buf)1057 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1058 {
1059 	int i = 0;
1060 	const char *p;
1061 	const int max_netdevices = 8*PAGE_SIZE;
1062 	unsigned long *inuse;
1063 	struct net_device *d;
1064 
1065 	p = strnchr(name, IFNAMSIZ-1, '%');
1066 	if (p) {
1067 		/*
1068 		 * Verify the string as this thing may have come from
1069 		 * the user.  There must be either one "%d" and no other "%"
1070 		 * characters.
1071 		 */
1072 		if (p[1] != 'd' || strchr(p + 2, '%'))
1073 			return -EINVAL;
1074 
1075 		/* Use one page as a bit array of possible slots */
1076 		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1077 		if (!inuse)
1078 			return -ENOMEM;
1079 
1080 		for_each_netdev(net, d) {
1081 			if (!sscanf(d->name, name, &i))
1082 				continue;
1083 			if (i < 0 || i >= max_netdevices)
1084 				continue;
1085 
1086 			/*  avoid cases where sscanf is not exact inverse of printf */
1087 			snprintf(buf, IFNAMSIZ, name, i);
1088 			if (!strncmp(buf, d->name, IFNAMSIZ))
1089 				set_bit(i, inuse);
1090 		}
1091 
1092 		i = find_first_zero_bit(inuse, max_netdevices);
1093 		free_page((unsigned long) inuse);
1094 	}
1095 
1096 	if (buf != name)
1097 		snprintf(buf, IFNAMSIZ, name, i);
1098 	if (!__dev_get_by_name(net, buf))
1099 		return i;
1100 
1101 	/* It is possible to run out of possible slots
1102 	 * when the name is long and there isn't enough space left
1103 	 * for the digits, or if all bits are used.
1104 	 */
1105 	return -ENFILE;
1106 }
1107 
1108 /**
1109  *	dev_alloc_name - allocate a name for a device
1110  *	@dev: device
1111  *	@name: name format string
1112  *
1113  *	Passed a format string - eg "lt%d" it will try and find a suitable
1114  *	id. It scans list of devices to build up a free map, then chooses
1115  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1116  *	while allocating the name and adding the device in order to avoid
1117  *	duplicates.
1118  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1119  *	Returns the number of the unit assigned or a negative errno code.
1120  */
1121 
dev_alloc_name(struct net_device * dev,const char * name)1122 int dev_alloc_name(struct net_device *dev, const char *name)
1123 {
1124 	char buf[IFNAMSIZ];
1125 	struct net *net;
1126 	int ret;
1127 
1128 	BUG_ON(!dev_net(dev));
1129 	net = dev_net(dev);
1130 	ret = __dev_alloc_name(net, name, buf);
1131 	if (ret >= 0)
1132 		strlcpy(dev->name, buf, IFNAMSIZ);
1133 	return ret;
1134 }
1135 EXPORT_SYMBOL(dev_alloc_name);
1136 
dev_alloc_name_ns(struct net * net,struct net_device * dev,const char * name)1137 static int dev_alloc_name_ns(struct net *net,
1138 			     struct net_device *dev,
1139 			     const char *name)
1140 {
1141 	char buf[IFNAMSIZ];
1142 	int ret;
1143 
1144 	ret = __dev_alloc_name(net, name, buf);
1145 	if (ret >= 0)
1146 		strlcpy(dev->name, buf, IFNAMSIZ);
1147 	return ret;
1148 }
1149 
dev_get_valid_name(struct net * net,struct net_device * dev,const char * name)1150 int dev_get_valid_name(struct net *net, struct net_device *dev,
1151 		       const char *name)
1152 {
1153 	BUG_ON(!net);
1154 
1155 	if (!dev_valid_name(name))
1156 		return -EINVAL;
1157 
1158 	if (strchr(name, '%'))
1159 		return dev_alloc_name_ns(net, dev, name);
1160 	else if (__dev_get_by_name(net, name))
1161 		return -EEXIST;
1162 	else if (dev->name != name)
1163 		strlcpy(dev->name, name, IFNAMSIZ);
1164 
1165 	return 0;
1166 }
1167 EXPORT_SYMBOL(dev_get_valid_name);
1168 
1169 /**
1170  *	dev_change_name - change name of a device
1171  *	@dev: device
1172  *	@newname: name (or format string) must be at least IFNAMSIZ
1173  *
1174  *	Change name of a device, can pass format strings "eth%d".
1175  *	for wildcarding.
1176  */
dev_change_name(struct net_device * dev,const char * newname)1177 int dev_change_name(struct net_device *dev, const char *newname)
1178 {
1179 	unsigned char old_assign_type;
1180 	char oldname[IFNAMSIZ];
1181 	int err = 0;
1182 	int ret;
1183 	struct net *net;
1184 
1185 	ASSERT_RTNL();
1186 	BUG_ON(!dev_net(dev));
1187 
1188 	net = dev_net(dev);
1189 	if (dev->flags & IFF_UP)
1190 		return -EBUSY;
1191 
1192 	write_seqcount_begin(&devnet_rename_seq);
1193 
1194 	if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1195 		write_seqcount_end(&devnet_rename_seq);
1196 		return 0;
1197 	}
1198 
1199 	memcpy(oldname, dev->name, IFNAMSIZ);
1200 
1201 	err = dev_get_valid_name(net, dev, newname);
1202 	if (err < 0) {
1203 		write_seqcount_end(&devnet_rename_seq);
1204 		return err;
1205 	}
1206 
1207 	if (oldname[0] && !strchr(oldname, '%'))
1208 		netdev_info(dev, "renamed from %s\n", oldname);
1209 
1210 	old_assign_type = dev->name_assign_type;
1211 	dev->name_assign_type = NET_NAME_RENAMED;
1212 
1213 rollback:
1214 	ret = device_rename(&dev->dev, dev->name);
1215 	if (ret) {
1216 		memcpy(dev->name, oldname, IFNAMSIZ);
1217 		dev->name_assign_type = old_assign_type;
1218 		write_seqcount_end(&devnet_rename_seq);
1219 		return ret;
1220 	}
1221 
1222 	write_seqcount_end(&devnet_rename_seq);
1223 
1224 	netdev_adjacent_rename_links(dev, oldname);
1225 
1226 	write_lock_bh(&dev_base_lock);
1227 	hlist_del_rcu(&dev->name_hlist);
1228 	write_unlock_bh(&dev_base_lock);
1229 
1230 	synchronize_rcu();
1231 
1232 	write_lock_bh(&dev_base_lock);
1233 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1234 	write_unlock_bh(&dev_base_lock);
1235 
1236 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1237 	ret = notifier_to_errno(ret);
1238 
1239 	if (ret) {
1240 		/* err >= 0 after dev_alloc_name() or stores the first errno */
1241 		if (err >= 0) {
1242 			err = ret;
1243 			write_seqcount_begin(&devnet_rename_seq);
1244 			memcpy(dev->name, oldname, IFNAMSIZ);
1245 			memcpy(oldname, newname, IFNAMSIZ);
1246 			dev->name_assign_type = old_assign_type;
1247 			old_assign_type = NET_NAME_RENAMED;
1248 			goto rollback;
1249 		} else {
1250 			pr_err("%s: name change rollback failed: %d\n",
1251 			       dev->name, ret);
1252 		}
1253 	}
1254 
1255 	return err;
1256 }
1257 
1258 /**
1259  *	dev_set_alias - change ifalias of a device
1260  *	@dev: device
1261  *	@alias: name up to IFALIASZ
1262  *	@len: limit of bytes to copy from info
1263  *
1264  *	Set ifalias for a device,
1265  */
dev_set_alias(struct net_device * dev,const char * alias,size_t len)1266 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1267 {
1268 	char *new_ifalias;
1269 
1270 	ASSERT_RTNL();
1271 
1272 	if (len >= IFALIASZ)
1273 		return -EINVAL;
1274 
1275 	if (!len) {
1276 		kfree(dev->ifalias);
1277 		dev->ifalias = NULL;
1278 		return 0;
1279 	}
1280 
1281 	new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1282 	if (!new_ifalias)
1283 		return -ENOMEM;
1284 	dev->ifalias = new_ifalias;
1285 	memcpy(dev->ifalias, alias, len);
1286 	dev->ifalias[len] = 0;
1287 
1288 	return len;
1289 }
1290 
1291 
1292 /**
1293  *	netdev_features_change - device changes features
1294  *	@dev: device to cause notification
1295  *
1296  *	Called to indicate a device has changed features.
1297  */
netdev_features_change(struct net_device * dev)1298 void netdev_features_change(struct net_device *dev)
1299 {
1300 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1301 }
1302 EXPORT_SYMBOL(netdev_features_change);
1303 
1304 /**
1305  *	netdev_state_change - device changes state
1306  *	@dev: device to cause notification
1307  *
1308  *	Called to indicate a device has changed state. This function calls
1309  *	the notifier chains for netdev_chain and sends a NEWLINK message
1310  *	to the routing socket.
1311  */
netdev_state_change(struct net_device * dev)1312 void netdev_state_change(struct net_device *dev)
1313 {
1314 	if (dev->flags & IFF_UP) {
1315 		struct netdev_notifier_change_info change_info;
1316 
1317 		change_info.flags_changed = 0;
1318 		call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1319 					      &change_info.info);
1320 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1321 	}
1322 }
1323 EXPORT_SYMBOL(netdev_state_change);
1324 
1325 /**
1326  * netdev_notify_peers - notify network peers about existence of @dev
1327  * @dev: network device
1328  *
1329  * Generate traffic such that interested network peers are aware of
1330  * @dev, such as by generating a gratuitous ARP. This may be used when
1331  * a device wants to inform the rest of the network about some sort of
1332  * reconfiguration such as a failover event or virtual machine
1333  * migration.
1334  */
netdev_notify_peers(struct net_device * dev)1335 void netdev_notify_peers(struct net_device *dev)
1336 {
1337 	rtnl_lock();
1338 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1339 	call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1340 	rtnl_unlock();
1341 }
1342 EXPORT_SYMBOL(netdev_notify_peers);
1343 
__dev_open(struct net_device * dev)1344 static int __dev_open(struct net_device *dev)
1345 {
1346 	const struct net_device_ops *ops = dev->netdev_ops;
1347 	int ret;
1348 
1349 	ASSERT_RTNL();
1350 
1351 	if (!netif_device_present(dev))
1352 		return -ENODEV;
1353 
1354 	/* Block netpoll from trying to do any rx path servicing.
1355 	 * If we don't do this there is a chance ndo_poll_controller
1356 	 * or ndo_poll may be running while we open the device
1357 	 */
1358 	netpoll_poll_disable(dev);
1359 
1360 	ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1361 	ret = notifier_to_errno(ret);
1362 	if (ret)
1363 		return ret;
1364 
1365 	set_bit(__LINK_STATE_START, &dev->state);
1366 
1367 	if (ops->ndo_validate_addr)
1368 		ret = ops->ndo_validate_addr(dev);
1369 
1370 	if (!ret && ops->ndo_open)
1371 		ret = ops->ndo_open(dev);
1372 
1373 	netpoll_poll_enable(dev);
1374 
1375 	if (ret)
1376 		clear_bit(__LINK_STATE_START, &dev->state);
1377 	else {
1378 		dev->flags |= IFF_UP;
1379 		dev_set_rx_mode(dev);
1380 		dev_activate(dev);
1381 		add_device_randomness(dev->dev_addr, dev->addr_len);
1382 	}
1383 
1384 	return ret;
1385 }
1386 
1387 /**
1388  *	dev_open	- prepare an interface for use.
1389  *	@dev:	device to open
1390  *
1391  *	Takes a device from down to up state. The device's private open
1392  *	function is invoked and then the multicast lists are loaded. Finally
1393  *	the device is moved into the up state and a %NETDEV_UP message is
1394  *	sent to the netdev notifier chain.
1395  *
1396  *	Calling this function on an active interface is a nop. On a failure
1397  *	a negative errno code is returned.
1398  */
dev_open(struct net_device * dev)1399 int dev_open(struct net_device *dev)
1400 {
1401 	int ret;
1402 
1403 	if (dev->flags & IFF_UP)
1404 		return 0;
1405 
1406 	ret = __dev_open(dev);
1407 	if (ret < 0)
1408 		return ret;
1409 
1410 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1411 	call_netdevice_notifiers(NETDEV_UP, dev);
1412 
1413 	return ret;
1414 }
1415 EXPORT_SYMBOL(dev_open);
1416 
__dev_close_many(struct list_head * head)1417 static void __dev_close_many(struct list_head *head)
1418 {
1419 	struct net_device *dev;
1420 
1421 	ASSERT_RTNL();
1422 	might_sleep();
1423 
1424 	list_for_each_entry(dev, head, close_list) {
1425 		/* Temporarily disable netpoll until the interface is down */
1426 		netpoll_poll_disable(dev);
1427 
1428 		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1429 
1430 		clear_bit(__LINK_STATE_START, &dev->state);
1431 
1432 		/* Synchronize to scheduled poll. We cannot touch poll list, it
1433 		 * can be even on different cpu. So just clear netif_running().
1434 		 *
1435 		 * dev->stop() will invoke napi_disable() on all of it's
1436 		 * napi_struct instances on this device.
1437 		 */
1438 		smp_mb__after_atomic(); /* Commit netif_running(). */
1439 	}
1440 
1441 	dev_deactivate_many(head);
1442 
1443 	list_for_each_entry(dev, head, close_list) {
1444 		const struct net_device_ops *ops = dev->netdev_ops;
1445 
1446 		/*
1447 		 *	Call the device specific close. This cannot fail.
1448 		 *	Only if device is UP
1449 		 *
1450 		 *	We allow it to be called even after a DETACH hot-plug
1451 		 *	event.
1452 		 */
1453 		if (ops->ndo_stop)
1454 			ops->ndo_stop(dev);
1455 
1456 		dev->flags &= ~IFF_UP;
1457 		netpoll_poll_enable(dev);
1458 	}
1459 }
1460 
__dev_close(struct net_device * dev)1461 static void __dev_close(struct net_device *dev)
1462 {
1463 	LIST_HEAD(single);
1464 
1465 	list_add(&dev->close_list, &single);
1466 	__dev_close_many(&single);
1467 	list_del(&single);
1468 }
1469 
dev_close_many(struct list_head * head,bool unlink)1470 void dev_close_many(struct list_head *head, bool unlink)
1471 {
1472 	struct net_device *dev, *tmp;
1473 
1474 	/* Remove the devices that don't need to be closed */
1475 	list_for_each_entry_safe(dev, tmp, head, close_list)
1476 		if (!(dev->flags & IFF_UP))
1477 			list_del_init(&dev->close_list);
1478 
1479 	__dev_close_many(head);
1480 
1481 	list_for_each_entry_safe(dev, tmp, head, close_list) {
1482 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1483 		call_netdevice_notifiers(NETDEV_DOWN, dev);
1484 		if (unlink)
1485 			list_del_init(&dev->close_list);
1486 	}
1487 }
1488 EXPORT_SYMBOL(dev_close_many);
1489 
1490 /**
1491  *	dev_close - shutdown an interface.
1492  *	@dev: device to shutdown
1493  *
1494  *	This function moves an active device into down state. A
1495  *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1496  *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1497  *	chain.
1498  */
dev_close(struct net_device * dev)1499 void dev_close(struct net_device *dev)
1500 {
1501 	if (dev->flags & IFF_UP) {
1502 		LIST_HEAD(single);
1503 
1504 		list_add(&dev->close_list, &single);
1505 		dev_close_many(&single, true);
1506 		list_del(&single);
1507 	}
1508 }
1509 EXPORT_SYMBOL(dev_close);
1510 
1511 
1512 /**
1513  *	dev_disable_lro - disable Large Receive Offload on a device
1514  *	@dev: device
1515  *
1516  *	Disable Large Receive Offload (LRO) on a net device.  Must be
1517  *	called under RTNL.  This is needed if received packets may be
1518  *	forwarded to another interface.
1519  */
dev_disable_lro(struct net_device * dev)1520 void dev_disable_lro(struct net_device *dev)
1521 {
1522 	struct net_device *lower_dev;
1523 	struct list_head *iter;
1524 
1525 	dev->wanted_features &= ~NETIF_F_LRO;
1526 	netdev_update_features(dev);
1527 
1528 	if (unlikely(dev->features & NETIF_F_LRO))
1529 		netdev_WARN(dev, "failed to disable LRO!\n");
1530 
1531 	netdev_for_each_lower_dev(dev, lower_dev, iter)
1532 		dev_disable_lro(lower_dev);
1533 }
1534 EXPORT_SYMBOL(dev_disable_lro);
1535 
call_netdevice_notifier(struct notifier_block * nb,unsigned long val,struct net_device * dev)1536 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1537 				   struct net_device *dev)
1538 {
1539 	struct netdev_notifier_info info;
1540 
1541 	netdev_notifier_info_init(&info, dev);
1542 	return nb->notifier_call(nb, val, &info);
1543 }
1544 
1545 static int dev_boot_phase = 1;
1546 
1547 /**
1548  * register_netdevice_notifier - register a network notifier block
1549  * @nb: notifier
1550  *
1551  * Register a notifier to be called when network device events occur.
1552  * The notifier passed is linked into the kernel structures and must
1553  * not be reused until it has been unregistered. A negative errno code
1554  * is returned on a failure.
1555  *
1556  * When registered all registration and up events are replayed
1557  * to the new notifier to allow device to have a race free
1558  * view of the network device list.
1559  */
1560 
register_netdevice_notifier(struct notifier_block * nb)1561 int register_netdevice_notifier(struct notifier_block *nb)
1562 {
1563 	struct net_device *dev;
1564 	struct net_device *last;
1565 	struct net *net;
1566 	int err;
1567 
1568 	rtnl_lock();
1569 	err = raw_notifier_chain_register(&netdev_chain, nb);
1570 	if (err)
1571 		goto unlock;
1572 	if (dev_boot_phase)
1573 		goto unlock;
1574 	for_each_net(net) {
1575 		for_each_netdev(net, dev) {
1576 			err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1577 			err = notifier_to_errno(err);
1578 			if (err)
1579 				goto rollback;
1580 
1581 			if (!(dev->flags & IFF_UP))
1582 				continue;
1583 
1584 			call_netdevice_notifier(nb, NETDEV_UP, dev);
1585 		}
1586 	}
1587 
1588 unlock:
1589 	rtnl_unlock();
1590 	return err;
1591 
1592 rollback:
1593 	last = dev;
1594 	for_each_net(net) {
1595 		for_each_netdev(net, dev) {
1596 			if (dev == last)
1597 				goto outroll;
1598 
1599 			if (dev->flags & IFF_UP) {
1600 				call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1601 							dev);
1602 				call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1603 			}
1604 			call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1605 		}
1606 	}
1607 
1608 outroll:
1609 	raw_notifier_chain_unregister(&netdev_chain, nb);
1610 	goto unlock;
1611 }
1612 EXPORT_SYMBOL(register_netdevice_notifier);
1613 
1614 /**
1615  * unregister_netdevice_notifier - unregister a network notifier block
1616  * @nb: notifier
1617  *
1618  * Unregister a notifier previously registered by
1619  * register_netdevice_notifier(). The notifier is unlinked into the
1620  * kernel structures and may then be reused. A negative errno code
1621  * is returned on a failure.
1622  *
1623  * After unregistering unregister and down device events are synthesized
1624  * for all devices on the device list to the removed notifier to remove
1625  * the need for special case cleanup code.
1626  */
1627 
unregister_netdevice_notifier(struct notifier_block * nb)1628 int unregister_netdevice_notifier(struct notifier_block *nb)
1629 {
1630 	struct net_device *dev;
1631 	struct net *net;
1632 	int err;
1633 
1634 	rtnl_lock();
1635 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1636 	if (err)
1637 		goto unlock;
1638 
1639 	for_each_net(net) {
1640 		for_each_netdev(net, dev) {
1641 			if (dev->flags & IFF_UP) {
1642 				call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1643 							dev);
1644 				call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1645 			}
1646 			call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1647 		}
1648 	}
1649 unlock:
1650 	rtnl_unlock();
1651 	return err;
1652 }
1653 EXPORT_SYMBOL(unregister_netdevice_notifier);
1654 
1655 /**
1656  *	call_netdevice_notifiers_info - call all network notifier blocks
1657  *	@val: value passed unmodified to notifier function
1658  *	@dev: net_device pointer passed unmodified to notifier function
1659  *	@info: notifier information data
1660  *
1661  *	Call all network notifier blocks.  Parameters and return value
1662  *	are as for raw_notifier_call_chain().
1663  */
1664 
call_netdevice_notifiers_info(unsigned long val,struct net_device * dev,struct netdev_notifier_info * info)1665 static int call_netdevice_notifiers_info(unsigned long val,
1666 					 struct net_device *dev,
1667 					 struct netdev_notifier_info *info)
1668 {
1669 	ASSERT_RTNL();
1670 	netdev_notifier_info_init(info, dev);
1671 	return raw_notifier_call_chain(&netdev_chain, val, info);
1672 }
1673 
1674 /**
1675  *	call_netdevice_notifiers - call all network notifier blocks
1676  *      @val: value passed unmodified to notifier function
1677  *      @dev: net_device pointer passed unmodified to notifier function
1678  *
1679  *	Call all network notifier blocks.  Parameters and return value
1680  *	are as for raw_notifier_call_chain().
1681  */
1682 
call_netdevice_notifiers(unsigned long val,struct net_device * dev)1683 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1684 {
1685 	struct netdev_notifier_info info;
1686 
1687 	return call_netdevice_notifiers_info(val, dev, &info);
1688 }
1689 EXPORT_SYMBOL(call_netdevice_notifiers);
1690 
1691 /**
1692  *	call_netdevice_notifiers_mtu - call all network notifier blocks
1693  *	@val: value passed unmodified to notifier function
1694  *	@dev: net_device pointer passed unmodified to notifier function
1695  *	@arg: additional u32 argument passed to the notifier function
1696  *
1697  *	Call all network notifier blocks.  Parameters and return value
1698  *	are as for raw_notifier_call_chain().
1699  */
call_netdevice_notifiers_mtu(unsigned long val,struct net_device * dev,u32 arg)1700 static int call_netdevice_notifiers_mtu(unsigned long val,
1701 					struct net_device *dev, u32 arg)
1702 {
1703 	struct netdev_notifier_info_ext info = {
1704 		.info.dev = dev,
1705 		.ext.mtu = arg,
1706 	};
1707 
1708 	BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
1709 
1710 	return call_netdevice_notifiers_info(val, dev, &info.info);
1711 }
1712 
1713 #ifdef CONFIG_NET_INGRESS
1714 static struct static_key ingress_needed __read_mostly;
1715 
net_inc_ingress_queue(void)1716 void net_inc_ingress_queue(void)
1717 {
1718 	static_key_slow_inc(&ingress_needed);
1719 }
1720 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1721 
net_dec_ingress_queue(void)1722 void net_dec_ingress_queue(void)
1723 {
1724 	static_key_slow_dec(&ingress_needed);
1725 }
1726 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1727 #endif
1728 
1729 #ifdef CONFIG_NET_EGRESS
1730 static struct static_key egress_needed __read_mostly;
1731 
net_inc_egress_queue(void)1732 void net_inc_egress_queue(void)
1733 {
1734 	static_key_slow_inc(&egress_needed);
1735 }
1736 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1737 
net_dec_egress_queue(void)1738 void net_dec_egress_queue(void)
1739 {
1740 	static_key_slow_dec(&egress_needed);
1741 }
1742 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1743 #endif
1744 
1745 static struct static_key netstamp_needed __read_mostly;
1746 #ifdef HAVE_JUMP_LABEL
1747 static atomic_t netstamp_needed_deferred;
1748 static atomic_t netstamp_wanted;
netstamp_clear(struct work_struct * work)1749 static void netstamp_clear(struct work_struct *work)
1750 {
1751 	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1752 	int wanted;
1753 
1754 	wanted = atomic_add_return(deferred, &netstamp_wanted);
1755 	if (wanted > 0)
1756 		static_key_enable(&netstamp_needed);
1757 	else
1758 		static_key_disable(&netstamp_needed);
1759 }
1760 static DECLARE_WORK(netstamp_work, netstamp_clear);
1761 #endif
1762 
net_enable_timestamp(void)1763 void net_enable_timestamp(void)
1764 {
1765 #ifdef HAVE_JUMP_LABEL
1766 	int wanted;
1767 
1768 	while (1) {
1769 		wanted = atomic_read(&netstamp_wanted);
1770 		if (wanted <= 0)
1771 			break;
1772 		if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1773 			return;
1774 	}
1775 	atomic_inc(&netstamp_needed_deferred);
1776 	schedule_work(&netstamp_work);
1777 #else
1778 	static_key_slow_inc(&netstamp_needed);
1779 #endif
1780 }
1781 EXPORT_SYMBOL(net_enable_timestamp);
1782 
net_disable_timestamp(void)1783 void net_disable_timestamp(void)
1784 {
1785 #ifdef HAVE_JUMP_LABEL
1786 	int wanted;
1787 
1788 	while (1) {
1789 		wanted = atomic_read(&netstamp_wanted);
1790 		if (wanted <= 1)
1791 			break;
1792 		if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1793 			return;
1794 	}
1795 	atomic_dec(&netstamp_needed_deferred);
1796 	schedule_work(&netstamp_work);
1797 #else
1798 	static_key_slow_dec(&netstamp_needed);
1799 #endif
1800 }
1801 EXPORT_SYMBOL(net_disable_timestamp);
1802 
net_timestamp_set(struct sk_buff * skb)1803 static inline void net_timestamp_set(struct sk_buff *skb)
1804 {
1805 	skb->tstamp = 0;
1806 	if (static_key_false(&netstamp_needed))
1807 		__net_timestamp(skb);
1808 }
1809 
1810 #define net_timestamp_check(COND, SKB)			\
1811 	if (static_key_false(&netstamp_needed)) {		\
1812 		if ((COND) && !(SKB)->tstamp)	\
1813 			__net_timestamp(SKB);		\
1814 	}						\
1815 
is_skb_forwardable(const struct net_device * dev,const struct sk_buff * skb)1816 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1817 {
1818 	unsigned int len;
1819 
1820 	if (!(dev->flags & IFF_UP))
1821 		return false;
1822 
1823 	len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1824 	if (skb->len <= len)
1825 		return true;
1826 
1827 	/* if TSO is enabled, we don't care about the length as the packet
1828 	 * could be forwarded without being segmented before
1829 	 */
1830 	if (skb_is_gso(skb))
1831 		return true;
1832 
1833 	return false;
1834 }
1835 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1836 
__dev_forward_skb(struct net_device * dev,struct sk_buff * skb)1837 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1838 {
1839 	int ret = ____dev_forward_skb(dev, skb);
1840 
1841 	if (likely(!ret)) {
1842 		skb->protocol = eth_type_trans(skb, dev);
1843 		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1844 	}
1845 
1846 	return ret;
1847 }
1848 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1849 
1850 /**
1851  * dev_forward_skb - loopback an skb to another netif
1852  *
1853  * @dev: destination network device
1854  * @skb: buffer to forward
1855  *
1856  * return values:
1857  *	NET_RX_SUCCESS	(no congestion)
1858  *	NET_RX_DROP     (packet was dropped, but freed)
1859  *
1860  * dev_forward_skb can be used for injecting an skb from the
1861  * start_xmit function of one device into the receive queue
1862  * of another device.
1863  *
1864  * The receiving device may be in another namespace, so
1865  * we have to clear all information in the skb that could
1866  * impact namespace isolation.
1867  */
dev_forward_skb(struct net_device * dev,struct sk_buff * skb)1868 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1869 {
1870 	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1871 }
1872 EXPORT_SYMBOL_GPL(dev_forward_skb);
1873 
deliver_skb(struct sk_buff * skb,struct packet_type * pt_prev,struct net_device * orig_dev)1874 static inline int deliver_skb(struct sk_buff *skb,
1875 			      struct packet_type *pt_prev,
1876 			      struct net_device *orig_dev)
1877 {
1878 	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1879 		return -ENOMEM;
1880 	refcount_inc(&skb->users);
1881 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1882 }
1883 
deliver_ptype_list_skb(struct sk_buff * skb,struct packet_type ** pt,struct net_device * orig_dev,__be16 type,struct list_head * ptype_list)1884 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1885 					  struct packet_type **pt,
1886 					  struct net_device *orig_dev,
1887 					  __be16 type,
1888 					  struct list_head *ptype_list)
1889 {
1890 	struct packet_type *ptype, *pt_prev = *pt;
1891 
1892 	list_for_each_entry_rcu(ptype, ptype_list, list) {
1893 		if (ptype->type != type)
1894 			continue;
1895 		if (pt_prev)
1896 			deliver_skb(skb, pt_prev, orig_dev);
1897 		pt_prev = ptype;
1898 	}
1899 	*pt = pt_prev;
1900 }
1901 
skb_loop_sk(struct packet_type * ptype,struct sk_buff * skb)1902 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1903 {
1904 	if (!ptype->af_packet_priv || !skb->sk)
1905 		return false;
1906 
1907 	if (ptype->id_match)
1908 		return ptype->id_match(ptype, skb->sk);
1909 	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1910 		return true;
1911 
1912 	return false;
1913 }
1914 
1915 /*
1916  *	Support routine. Sends outgoing frames to any network
1917  *	taps currently in use.
1918  */
1919 
dev_queue_xmit_nit(struct sk_buff * skb,struct net_device * dev)1920 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1921 {
1922 	struct packet_type *ptype;
1923 	struct sk_buff *skb2 = NULL;
1924 	struct packet_type *pt_prev = NULL;
1925 	struct list_head *ptype_list = &ptype_all;
1926 
1927 	rcu_read_lock();
1928 again:
1929 	list_for_each_entry_rcu(ptype, ptype_list, list) {
1930 		/* Never send packets back to the socket
1931 		 * they originated from - MvS (miquels@drinkel.ow.org)
1932 		 */
1933 		if (skb_loop_sk(ptype, skb))
1934 			continue;
1935 
1936 		if (pt_prev) {
1937 			deliver_skb(skb2, pt_prev, skb->dev);
1938 			pt_prev = ptype;
1939 			continue;
1940 		}
1941 
1942 		/* need to clone skb, done only once */
1943 		skb2 = skb_clone(skb, GFP_ATOMIC);
1944 		if (!skb2)
1945 			goto out_unlock;
1946 
1947 		net_timestamp_set(skb2);
1948 
1949 		/* skb->nh should be correctly
1950 		 * set by sender, so that the second statement is
1951 		 * just protection against buggy protocols.
1952 		 */
1953 		skb_reset_mac_header(skb2);
1954 
1955 		if (skb_network_header(skb2) < skb2->data ||
1956 		    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1957 			net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1958 					     ntohs(skb2->protocol),
1959 					     dev->name);
1960 			skb_reset_network_header(skb2);
1961 		}
1962 
1963 		skb2->transport_header = skb2->network_header;
1964 		skb2->pkt_type = PACKET_OUTGOING;
1965 		pt_prev = ptype;
1966 	}
1967 
1968 	if (ptype_list == &ptype_all) {
1969 		ptype_list = &dev->ptype_all;
1970 		goto again;
1971 	}
1972 out_unlock:
1973 	if (pt_prev) {
1974 		if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
1975 			pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1976 		else
1977 			kfree_skb(skb2);
1978 	}
1979 	rcu_read_unlock();
1980 }
1981 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1982 
1983 /**
1984  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1985  * @dev: Network device
1986  * @txq: number of queues available
1987  *
1988  * If real_num_tx_queues is changed the tc mappings may no longer be
1989  * valid. To resolve this verify the tc mapping remains valid and if
1990  * not NULL the mapping. With no priorities mapping to this
1991  * offset/count pair it will no longer be used. In the worst case TC0
1992  * is invalid nothing can be done so disable priority mappings. If is
1993  * expected that drivers will fix this mapping if they can before
1994  * calling netif_set_real_num_tx_queues.
1995  */
netif_setup_tc(struct net_device * dev,unsigned int txq)1996 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1997 {
1998 	int i;
1999 	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2000 
2001 	/* If TC0 is invalidated disable TC mapping */
2002 	if (tc->offset + tc->count > txq) {
2003 		pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2004 		dev->num_tc = 0;
2005 		return;
2006 	}
2007 
2008 	/* Invalidated prio to tc mappings set to TC0 */
2009 	for (i = 1; i < TC_BITMASK + 1; i++) {
2010 		int q = netdev_get_prio_tc_map(dev, i);
2011 
2012 		tc = &dev->tc_to_txq[q];
2013 		if (tc->offset + tc->count > txq) {
2014 			pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2015 				i, q);
2016 			netdev_set_prio_tc_map(dev, i, 0);
2017 		}
2018 	}
2019 }
2020 
netdev_txq_to_tc(struct net_device * dev,unsigned int txq)2021 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2022 {
2023 	if (dev->num_tc) {
2024 		struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2025 		int i;
2026 
2027 		for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2028 			if ((txq - tc->offset) < tc->count)
2029 				return i;
2030 		}
2031 
2032 		return -1;
2033 	}
2034 
2035 	return 0;
2036 }
2037 
2038 #ifdef CONFIG_XPS
2039 static DEFINE_MUTEX(xps_map_mutex);
2040 #define xmap_dereference(P)		\
2041 	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2042 
remove_xps_queue(struct xps_dev_maps * dev_maps,int tci,u16 index)2043 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2044 			     int tci, u16 index)
2045 {
2046 	struct xps_map *map = NULL;
2047 	int pos;
2048 
2049 	if (dev_maps)
2050 		map = xmap_dereference(dev_maps->cpu_map[tci]);
2051 	if (!map)
2052 		return false;
2053 
2054 	for (pos = map->len; pos--;) {
2055 		if (map->queues[pos] != index)
2056 			continue;
2057 
2058 		if (map->len > 1) {
2059 			map->queues[pos] = map->queues[--map->len];
2060 			break;
2061 		}
2062 
2063 		RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2064 		kfree_rcu(map, rcu);
2065 		return false;
2066 	}
2067 
2068 	return true;
2069 }
2070 
remove_xps_queue_cpu(struct net_device * dev,struct xps_dev_maps * dev_maps,int cpu,u16 offset,u16 count)2071 static bool remove_xps_queue_cpu(struct net_device *dev,
2072 				 struct xps_dev_maps *dev_maps,
2073 				 int cpu, u16 offset, u16 count)
2074 {
2075 	int num_tc = dev->num_tc ? : 1;
2076 	bool active = false;
2077 	int tci;
2078 
2079 	for (tci = cpu * num_tc; num_tc--; tci++) {
2080 		int i, j;
2081 
2082 		for (i = count, j = offset; i--; j++) {
2083 			if (!remove_xps_queue(dev_maps, tci, j))
2084 				break;
2085 		}
2086 
2087 		active |= i < 0;
2088 	}
2089 
2090 	return active;
2091 }
2092 
netif_reset_xps_queues(struct net_device * dev,u16 offset,u16 count)2093 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2094 				   u16 count)
2095 {
2096 	struct xps_dev_maps *dev_maps;
2097 	int cpu, i;
2098 	bool active = false;
2099 
2100 	mutex_lock(&xps_map_mutex);
2101 	dev_maps = xmap_dereference(dev->xps_maps);
2102 
2103 	if (!dev_maps)
2104 		goto out_no_maps;
2105 
2106 	for_each_possible_cpu(cpu)
2107 		active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2108 					       offset, count);
2109 
2110 	if (!active) {
2111 		RCU_INIT_POINTER(dev->xps_maps, NULL);
2112 		kfree_rcu(dev_maps, rcu);
2113 	}
2114 
2115 	for (i = offset + (count - 1); count--; i--)
2116 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2117 					     NUMA_NO_NODE);
2118 
2119 out_no_maps:
2120 	mutex_unlock(&xps_map_mutex);
2121 }
2122 
netif_reset_xps_queues_gt(struct net_device * dev,u16 index)2123 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2124 {
2125 	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2126 }
2127 
expand_xps_map(struct xps_map * map,int cpu,u16 index)2128 static struct xps_map *expand_xps_map(struct xps_map *map,
2129 				      int cpu, u16 index)
2130 {
2131 	struct xps_map *new_map;
2132 	int alloc_len = XPS_MIN_MAP_ALLOC;
2133 	int i, pos;
2134 
2135 	for (pos = 0; map && pos < map->len; pos++) {
2136 		if (map->queues[pos] != index)
2137 			continue;
2138 		return map;
2139 	}
2140 
2141 	/* Need to add queue to this CPU's existing map */
2142 	if (map) {
2143 		if (pos < map->alloc_len)
2144 			return map;
2145 
2146 		alloc_len = map->alloc_len * 2;
2147 	}
2148 
2149 	/* Need to allocate new map to store queue on this CPU's map */
2150 	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2151 			       cpu_to_node(cpu));
2152 	if (!new_map)
2153 		return NULL;
2154 
2155 	for (i = 0; i < pos; i++)
2156 		new_map->queues[i] = map->queues[i];
2157 	new_map->alloc_len = alloc_len;
2158 	new_map->len = pos;
2159 
2160 	return new_map;
2161 }
2162 
netif_set_xps_queue(struct net_device * dev,const struct cpumask * mask,u16 index)2163 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2164 			u16 index)
2165 {
2166 	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2167 	int i, cpu, tci, numa_node_id = -2;
2168 	int maps_sz, num_tc = 1, tc = 0;
2169 	struct xps_map *map, *new_map;
2170 	bool active = false;
2171 
2172 	if (dev->num_tc) {
2173 		num_tc = dev->num_tc;
2174 		tc = netdev_txq_to_tc(dev, index);
2175 		if (tc < 0)
2176 			return -EINVAL;
2177 	}
2178 
2179 	maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2180 	if (maps_sz < L1_CACHE_BYTES)
2181 		maps_sz = L1_CACHE_BYTES;
2182 
2183 	mutex_lock(&xps_map_mutex);
2184 
2185 	dev_maps = xmap_dereference(dev->xps_maps);
2186 
2187 	/* allocate memory for queue storage */
2188 	for_each_cpu_and(cpu, cpu_online_mask, mask) {
2189 		if (!new_dev_maps)
2190 			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2191 		if (!new_dev_maps) {
2192 			mutex_unlock(&xps_map_mutex);
2193 			return -ENOMEM;
2194 		}
2195 
2196 		tci = cpu * num_tc + tc;
2197 		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2198 				 NULL;
2199 
2200 		map = expand_xps_map(map, cpu, index);
2201 		if (!map)
2202 			goto error;
2203 
2204 		RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2205 	}
2206 
2207 	if (!new_dev_maps)
2208 		goto out_no_new_maps;
2209 
2210 	for_each_possible_cpu(cpu) {
2211 		/* copy maps belonging to foreign traffic classes */
2212 		for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2213 			/* fill in the new device map from the old device map */
2214 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2215 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2216 		}
2217 
2218 		/* We need to explicitly update tci as prevous loop
2219 		 * could break out early if dev_maps is NULL.
2220 		 */
2221 		tci = cpu * num_tc + tc;
2222 
2223 		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2224 			/* add queue to CPU maps */
2225 			int pos = 0;
2226 
2227 			map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2228 			while ((pos < map->len) && (map->queues[pos] != index))
2229 				pos++;
2230 
2231 			if (pos == map->len)
2232 				map->queues[map->len++] = index;
2233 #ifdef CONFIG_NUMA
2234 			if (numa_node_id == -2)
2235 				numa_node_id = cpu_to_node(cpu);
2236 			else if (numa_node_id != cpu_to_node(cpu))
2237 				numa_node_id = -1;
2238 #endif
2239 		} else if (dev_maps) {
2240 			/* fill in the new device map from the old device map */
2241 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2242 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2243 		}
2244 
2245 		/* copy maps belonging to foreign traffic classes */
2246 		for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2247 			/* fill in the new device map from the old device map */
2248 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2249 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2250 		}
2251 	}
2252 
2253 	rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2254 
2255 	/* Cleanup old maps */
2256 	if (!dev_maps)
2257 		goto out_no_old_maps;
2258 
2259 	for_each_possible_cpu(cpu) {
2260 		for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2261 			new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2262 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2263 			if (map && map != new_map)
2264 				kfree_rcu(map, rcu);
2265 		}
2266 	}
2267 
2268 	kfree_rcu(dev_maps, rcu);
2269 
2270 out_no_old_maps:
2271 	dev_maps = new_dev_maps;
2272 	active = true;
2273 
2274 out_no_new_maps:
2275 	/* update Tx queue numa node */
2276 	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2277 				     (numa_node_id >= 0) ? numa_node_id :
2278 				     NUMA_NO_NODE);
2279 
2280 	if (!dev_maps)
2281 		goto out_no_maps;
2282 
2283 	/* removes queue from unused CPUs */
2284 	for_each_possible_cpu(cpu) {
2285 		for (i = tc, tci = cpu * num_tc; i--; tci++)
2286 			active |= remove_xps_queue(dev_maps, tci, index);
2287 		if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2288 			active |= remove_xps_queue(dev_maps, tci, index);
2289 		for (i = num_tc - tc, tci++; --i; tci++)
2290 			active |= remove_xps_queue(dev_maps, tci, index);
2291 	}
2292 
2293 	/* free map if not active */
2294 	if (!active) {
2295 		RCU_INIT_POINTER(dev->xps_maps, NULL);
2296 		kfree_rcu(dev_maps, rcu);
2297 	}
2298 
2299 out_no_maps:
2300 	mutex_unlock(&xps_map_mutex);
2301 
2302 	return 0;
2303 error:
2304 	/* remove any maps that we added */
2305 	for_each_possible_cpu(cpu) {
2306 		for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2307 			new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2308 			map = dev_maps ?
2309 			      xmap_dereference(dev_maps->cpu_map[tci]) :
2310 			      NULL;
2311 			if (new_map && new_map != map)
2312 				kfree(new_map);
2313 		}
2314 	}
2315 
2316 	mutex_unlock(&xps_map_mutex);
2317 
2318 	kfree(new_dev_maps);
2319 	return -ENOMEM;
2320 }
2321 EXPORT_SYMBOL(netif_set_xps_queue);
2322 
2323 #endif
netdev_reset_tc(struct net_device * dev)2324 void netdev_reset_tc(struct net_device *dev)
2325 {
2326 #ifdef CONFIG_XPS
2327 	netif_reset_xps_queues_gt(dev, 0);
2328 #endif
2329 	dev->num_tc = 0;
2330 	memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2331 	memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2332 }
2333 EXPORT_SYMBOL(netdev_reset_tc);
2334 
netdev_set_tc_queue(struct net_device * dev,u8 tc,u16 count,u16 offset)2335 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2336 {
2337 	if (tc >= dev->num_tc)
2338 		return -EINVAL;
2339 
2340 #ifdef CONFIG_XPS
2341 	netif_reset_xps_queues(dev, offset, count);
2342 #endif
2343 	dev->tc_to_txq[tc].count = count;
2344 	dev->tc_to_txq[tc].offset = offset;
2345 	return 0;
2346 }
2347 EXPORT_SYMBOL(netdev_set_tc_queue);
2348 
netdev_set_num_tc(struct net_device * dev,u8 num_tc)2349 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2350 {
2351 	if (num_tc > TC_MAX_QUEUE)
2352 		return -EINVAL;
2353 
2354 #ifdef CONFIG_XPS
2355 	netif_reset_xps_queues_gt(dev, 0);
2356 #endif
2357 	dev->num_tc = num_tc;
2358 	return 0;
2359 }
2360 EXPORT_SYMBOL(netdev_set_num_tc);
2361 
2362 /*
2363  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2364  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2365  */
netif_set_real_num_tx_queues(struct net_device * dev,unsigned int txq)2366 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2367 {
2368 	bool disabling;
2369 	int rc;
2370 
2371 	disabling = txq < dev->real_num_tx_queues;
2372 
2373 	if (txq < 1 || txq > dev->num_tx_queues)
2374 		return -EINVAL;
2375 
2376 	if (dev->reg_state == NETREG_REGISTERED ||
2377 	    dev->reg_state == NETREG_UNREGISTERING) {
2378 		ASSERT_RTNL();
2379 
2380 		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2381 						  txq);
2382 		if (rc)
2383 			return rc;
2384 
2385 		if (dev->num_tc)
2386 			netif_setup_tc(dev, txq);
2387 
2388 		dev->real_num_tx_queues = txq;
2389 
2390 		if (disabling) {
2391 			synchronize_net();
2392 			qdisc_reset_all_tx_gt(dev, txq);
2393 #ifdef CONFIG_XPS
2394 			netif_reset_xps_queues_gt(dev, txq);
2395 #endif
2396 		}
2397 	} else {
2398 		dev->real_num_tx_queues = txq;
2399 	}
2400 
2401 	return 0;
2402 }
2403 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2404 
2405 #ifdef CONFIG_SYSFS
2406 /**
2407  *	netif_set_real_num_rx_queues - set actual number of RX queues used
2408  *	@dev: Network device
2409  *	@rxq: Actual number of RX queues
2410  *
2411  *	This must be called either with the rtnl_lock held or before
2412  *	registration of the net device.  Returns 0 on success, or a
2413  *	negative error code.  If called before registration, it always
2414  *	succeeds.
2415  */
netif_set_real_num_rx_queues(struct net_device * dev,unsigned int rxq)2416 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2417 {
2418 	int rc;
2419 
2420 	if (rxq < 1 || rxq > dev->num_rx_queues)
2421 		return -EINVAL;
2422 
2423 	if (dev->reg_state == NETREG_REGISTERED) {
2424 		ASSERT_RTNL();
2425 
2426 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2427 						  rxq);
2428 		if (rc)
2429 			return rc;
2430 	}
2431 
2432 	dev->real_num_rx_queues = rxq;
2433 	return 0;
2434 }
2435 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2436 #endif
2437 
2438 /**
2439  * netif_get_num_default_rss_queues - default number of RSS queues
2440  *
2441  * This routine should set an upper limit on the number of RSS queues
2442  * used by default by multiqueue devices.
2443  */
netif_get_num_default_rss_queues(void)2444 int netif_get_num_default_rss_queues(void)
2445 {
2446 	return is_kdump_kernel() ?
2447 		1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2448 }
2449 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2450 
__netif_reschedule(struct Qdisc * q)2451 static void __netif_reschedule(struct Qdisc *q)
2452 {
2453 	struct softnet_data *sd;
2454 	unsigned long flags;
2455 
2456 	local_irq_save(flags);
2457 	sd = this_cpu_ptr(&softnet_data);
2458 	q->next_sched = NULL;
2459 	*sd->output_queue_tailp = q;
2460 	sd->output_queue_tailp = &q->next_sched;
2461 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2462 	local_irq_restore(flags);
2463 }
2464 
__netif_schedule(struct Qdisc * q)2465 void __netif_schedule(struct Qdisc *q)
2466 {
2467 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2468 		__netif_reschedule(q);
2469 }
2470 EXPORT_SYMBOL(__netif_schedule);
2471 
2472 struct dev_kfree_skb_cb {
2473 	enum skb_free_reason reason;
2474 };
2475 
get_kfree_skb_cb(const struct sk_buff * skb)2476 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2477 {
2478 	return (struct dev_kfree_skb_cb *)skb->cb;
2479 }
2480 
netif_schedule_queue(struct netdev_queue * txq)2481 void netif_schedule_queue(struct netdev_queue *txq)
2482 {
2483 	rcu_read_lock();
2484 	if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2485 		struct Qdisc *q = rcu_dereference(txq->qdisc);
2486 
2487 		__netif_schedule(q);
2488 	}
2489 	rcu_read_unlock();
2490 }
2491 EXPORT_SYMBOL(netif_schedule_queue);
2492 
netif_tx_wake_queue(struct netdev_queue * dev_queue)2493 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2494 {
2495 	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2496 		struct Qdisc *q;
2497 
2498 		rcu_read_lock();
2499 		q = rcu_dereference(dev_queue->qdisc);
2500 		__netif_schedule(q);
2501 		rcu_read_unlock();
2502 	}
2503 }
2504 EXPORT_SYMBOL(netif_tx_wake_queue);
2505 
__dev_kfree_skb_irq(struct sk_buff * skb,enum skb_free_reason reason)2506 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2507 {
2508 	unsigned long flags;
2509 
2510 	if (unlikely(!skb))
2511 		return;
2512 
2513 	if (likely(refcount_read(&skb->users) == 1)) {
2514 		smp_rmb();
2515 		refcount_set(&skb->users, 0);
2516 	} else if (likely(!refcount_dec_and_test(&skb->users))) {
2517 		return;
2518 	}
2519 	get_kfree_skb_cb(skb)->reason = reason;
2520 	local_irq_save(flags);
2521 	skb->next = __this_cpu_read(softnet_data.completion_queue);
2522 	__this_cpu_write(softnet_data.completion_queue, skb);
2523 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2524 	local_irq_restore(flags);
2525 }
2526 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2527 
__dev_kfree_skb_any(struct sk_buff * skb,enum skb_free_reason reason)2528 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2529 {
2530 	if (in_irq() || irqs_disabled())
2531 		__dev_kfree_skb_irq(skb, reason);
2532 	else
2533 		dev_kfree_skb(skb);
2534 }
2535 EXPORT_SYMBOL(__dev_kfree_skb_any);
2536 
2537 
2538 /**
2539  * netif_device_detach - mark device as removed
2540  * @dev: network device
2541  *
2542  * Mark device as removed from system and therefore no longer available.
2543  */
netif_device_detach(struct net_device * dev)2544 void netif_device_detach(struct net_device *dev)
2545 {
2546 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2547 	    netif_running(dev)) {
2548 		netif_tx_stop_all_queues(dev);
2549 	}
2550 }
2551 EXPORT_SYMBOL(netif_device_detach);
2552 
2553 /**
2554  * netif_device_attach - mark device as attached
2555  * @dev: network device
2556  *
2557  * Mark device as attached from system and restart if needed.
2558  */
netif_device_attach(struct net_device * dev)2559 void netif_device_attach(struct net_device *dev)
2560 {
2561 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2562 	    netif_running(dev)) {
2563 		netif_tx_wake_all_queues(dev);
2564 		__netdev_watchdog_up(dev);
2565 	}
2566 }
2567 EXPORT_SYMBOL(netif_device_attach);
2568 
2569 /*
2570  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2571  * to be used as a distribution range.
2572  */
__skb_tx_hash(const struct net_device * dev,struct sk_buff * skb,unsigned int num_tx_queues)2573 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2574 		  unsigned int num_tx_queues)
2575 {
2576 	u32 hash;
2577 	u16 qoffset = 0;
2578 	u16 qcount = num_tx_queues;
2579 
2580 	if (skb_rx_queue_recorded(skb)) {
2581 		hash = skb_get_rx_queue(skb);
2582 		while (unlikely(hash >= num_tx_queues))
2583 			hash -= num_tx_queues;
2584 		return hash;
2585 	}
2586 
2587 	if (dev->num_tc) {
2588 		u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2589 
2590 		qoffset = dev->tc_to_txq[tc].offset;
2591 		qcount = dev->tc_to_txq[tc].count;
2592 	}
2593 
2594 	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2595 }
2596 EXPORT_SYMBOL(__skb_tx_hash);
2597 
skb_warn_bad_offload(const struct sk_buff * skb)2598 static void skb_warn_bad_offload(const struct sk_buff *skb)
2599 {
2600 	static const netdev_features_t null_features;
2601 	struct net_device *dev = skb->dev;
2602 	const char *name = "";
2603 
2604 	if (!net_ratelimit())
2605 		return;
2606 
2607 	if (dev) {
2608 		if (dev->dev.parent)
2609 			name = dev_driver_string(dev->dev.parent);
2610 		else
2611 			name = netdev_name(dev);
2612 	}
2613 	WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2614 	     "gso_type=%d ip_summed=%d\n",
2615 	     name, dev ? &dev->features : &null_features,
2616 	     skb->sk ? &skb->sk->sk_route_caps : &null_features,
2617 	     skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2618 	     skb_shinfo(skb)->gso_type, skb->ip_summed);
2619 }
2620 
2621 /*
2622  * Invalidate hardware checksum when packet is to be mangled, and
2623  * complete checksum manually on outgoing path.
2624  */
skb_checksum_help(struct sk_buff * skb)2625 int skb_checksum_help(struct sk_buff *skb)
2626 {
2627 	__wsum csum;
2628 	int ret = 0, offset;
2629 
2630 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2631 		goto out_set_summed;
2632 
2633 	if (unlikely(skb_shinfo(skb)->gso_size)) {
2634 		skb_warn_bad_offload(skb);
2635 		return -EINVAL;
2636 	}
2637 
2638 	/* Before computing a checksum, we should make sure no frag could
2639 	 * be modified by an external entity : checksum could be wrong.
2640 	 */
2641 	if (skb_has_shared_frag(skb)) {
2642 		ret = __skb_linearize(skb);
2643 		if (ret)
2644 			goto out;
2645 	}
2646 
2647 	offset = skb_checksum_start_offset(skb);
2648 	BUG_ON(offset >= skb_headlen(skb));
2649 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
2650 
2651 	offset += skb->csum_offset;
2652 	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2653 
2654 	if (skb_cloned(skb) &&
2655 	    !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2656 		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2657 		if (ret)
2658 			goto out;
2659 	}
2660 
2661 	*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2662 out_set_summed:
2663 	skb->ip_summed = CHECKSUM_NONE;
2664 out:
2665 	return ret;
2666 }
2667 EXPORT_SYMBOL(skb_checksum_help);
2668 
skb_crc32c_csum_help(struct sk_buff * skb)2669 int skb_crc32c_csum_help(struct sk_buff *skb)
2670 {
2671 	__le32 crc32c_csum;
2672 	int ret = 0, offset, start;
2673 
2674 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2675 		goto out;
2676 
2677 	if (unlikely(skb_is_gso(skb)))
2678 		goto out;
2679 
2680 	/* Before computing a checksum, we should make sure no frag could
2681 	 * be modified by an external entity : checksum could be wrong.
2682 	 */
2683 	if (unlikely(skb_has_shared_frag(skb))) {
2684 		ret = __skb_linearize(skb);
2685 		if (ret)
2686 			goto out;
2687 	}
2688 	start = skb_checksum_start_offset(skb);
2689 	offset = start + offsetof(struct sctphdr, checksum);
2690 	if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2691 		ret = -EINVAL;
2692 		goto out;
2693 	}
2694 	if (skb_cloned(skb) &&
2695 	    !skb_clone_writable(skb, offset + sizeof(__le32))) {
2696 		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2697 		if (ret)
2698 			goto out;
2699 	}
2700 	crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2701 						  skb->len - start, ~(__u32)0,
2702 						  crc32c_csum_stub));
2703 	*(__le32 *)(skb->data + offset) = crc32c_csum;
2704 	skb->ip_summed = CHECKSUM_NONE;
2705 	skb->csum_not_inet = 0;
2706 out:
2707 	return ret;
2708 }
2709 
skb_network_protocol(struct sk_buff * skb,int * depth)2710 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2711 {
2712 	__be16 type = skb->protocol;
2713 
2714 	/* Tunnel gso handlers can set protocol to ethernet. */
2715 	if (type == htons(ETH_P_TEB)) {
2716 		struct ethhdr *eth;
2717 
2718 		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2719 			return 0;
2720 
2721 		eth = (struct ethhdr *)skb->data;
2722 		type = eth->h_proto;
2723 	}
2724 
2725 	return __vlan_get_protocol(skb, type, depth);
2726 }
2727 
2728 /**
2729  *	skb_mac_gso_segment - mac layer segmentation handler.
2730  *	@skb: buffer to segment
2731  *	@features: features for the output path (see dev->features)
2732  */
skb_mac_gso_segment(struct sk_buff * skb,netdev_features_t features)2733 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2734 				    netdev_features_t features)
2735 {
2736 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2737 	struct packet_offload *ptype;
2738 	int vlan_depth = skb->mac_len;
2739 	__be16 type = skb_network_protocol(skb, &vlan_depth);
2740 
2741 	if (unlikely(!type))
2742 		return ERR_PTR(-EINVAL);
2743 
2744 	__skb_pull(skb, vlan_depth);
2745 
2746 	rcu_read_lock();
2747 	list_for_each_entry_rcu(ptype, &offload_base, list) {
2748 		if (ptype->type == type && ptype->callbacks.gso_segment) {
2749 			segs = ptype->callbacks.gso_segment(skb, features);
2750 			break;
2751 		}
2752 	}
2753 	rcu_read_unlock();
2754 
2755 	__skb_push(skb, skb->data - skb_mac_header(skb));
2756 
2757 	return segs;
2758 }
2759 EXPORT_SYMBOL(skb_mac_gso_segment);
2760 
2761 
2762 /* openvswitch calls this on rx path, so we need a different check.
2763  */
skb_needs_check(struct sk_buff * skb,bool tx_path)2764 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2765 {
2766 	if (tx_path)
2767 		return skb->ip_summed != CHECKSUM_PARTIAL &&
2768 		       skb->ip_summed != CHECKSUM_UNNECESSARY;
2769 
2770 	return skb->ip_summed == CHECKSUM_NONE;
2771 }
2772 
2773 /**
2774  *	__skb_gso_segment - Perform segmentation on skb.
2775  *	@skb: buffer to segment
2776  *	@features: features for the output path (see dev->features)
2777  *	@tx_path: whether it is called in TX path
2778  *
2779  *	This function segments the given skb and returns a list of segments.
2780  *
2781  *	It may return NULL if the skb requires no segmentation.  This is
2782  *	only possible when GSO is used for verifying header integrity.
2783  *
2784  *	Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2785  */
__skb_gso_segment(struct sk_buff * skb,netdev_features_t features,bool tx_path)2786 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2787 				  netdev_features_t features, bool tx_path)
2788 {
2789 	struct sk_buff *segs;
2790 
2791 	if (unlikely(skb_needs_check(skb, tx_path))) {
2792 		int err;
2793 
2794 		/* We're going to init ->check field in TCP or UDP header */
2795 		err = skb_cow_head(skb, 0);
2796 		if (err < 0)
2797 			return ERR_PTR(err);
2798 	}
2799 
2800 	/* Only report GSO partial support if it will enable us to
2801 	 * support segmentation on this frame without needing additional
2802 	 * work.
2803 	 */
2804 	if (features & NETIF_F_GSO_PARTIAL) {
2805 		netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2806 		struct net_device *dev = skb->dev;
2807 
2808 		partial_features |= dev->features & dev->gso_partial_features;
2809 		if (!skb_gso_ok(skb, features | partial_features))
2810 			features &= ~NETIF_F_GSO_PARTIAL;
2811 	}
2812 
2813 	BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2814 		     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2815 
2816 	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2817 	SKB_GSO_CB(skb)->encap_level = 0;
2818 
2819 	skb_reset_mac_header(skb);
2820 	skb_reset_mac_len(skb);
2821 
2822 	segs = skb_mac_gso_segment(skb, features);
2823 
2824 	if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
2825 		skb_warn_bad_offload(skb);
2826 
2827 	return segs;
2828 }
2829 EXPORT_SYMBOL(__skb_gso_segment);
2830 
2831 /* Take action when hardware reception checksum errors are detected. */
2832 #ifdef CONFIG_BUG
netdev_rx_csum_fault(struct net_device * dev)2833 void netdev_rx_csum_fault(struct net_device *dev)
2834 {
2835 	if (net_ratelimit()) {
2836 		pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2837 		dump_stack();
2838 	}
2839 }
2840 EXPORT_SYMBOL(netdev_rx_csum_fault);
2841 #endif
2842 
2843 /* Actually, we should eliminate this check as soon as we know, that:
2844  * 1. IOMMU is present and allows to map all the memory.
2845  * 2. No high memory really exists on this machine.
2846  */
2847 
illegal_highdma(struct net_device * dev,struct sk_buff * skb)2848 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2849 {
2850 #ifdef CONFIG_HIGHMEM
2851 	int i;
2852 
2853 	if (!(dev->features & NETIF_F_HIGHDMA)) {
2854 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2855 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2856 
2857 			if (PageHighMem(skb_frag_page(frag)))
2858 				return 1;
2859 		}
2860 	}
2861 
2862 	if (PCI_DMA_BUS_IS_PHYS) {
2863 		struct device *pdev = dev->dev.parent;
2864 
2865 		if (!pdev)
2866 			return 0;
2867 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2868 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2869 			dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2870 
2871 			if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2872 				return 1;
2873 		}
2874 	}
2875 #endif
2876 	return 0;
2877 }
2878 
2879 /* If MPLS offload request, verify we are testing hardware MPLS features
2880  * instead of standard features for the netdev.
2881  */
2882 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
net_mpls_features(struct sk_buff * skb,netdev_features_t features,__be16 type)2883 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2884 					   netdev_features_t features,
2885 					   __be16 type)
2886 {
2887 	if (eth_p_mpls(type))
2888 		features &= skb->dev->mpls_features;
2889 
2890 	return features;
2891 }
2892 #else
net_mpls_features(struct sk_buff * skb,netdev_features_t features,__be16 type)2893 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2894 					   netdev_features_t features,
2895 					   __be16 type)
2896 {
2897 	return features;
2898 }
2899 #endif
2900 
harmonize_features(struct sk_buff * skb,netdev_features_t features)2901 static netdev_features_t harmonize_features(struct sk_buff *skb,
2902 	netdev_features_t features)
2903 {
2904 	int tmp;
2905 	__be16 type;
2906 
2907 	type = skb_network_protocol(skb, &tmp);
2908 	features = net_mpls_features(skb, features, type);
2909 
2910 	if (skb->ip_summed != CHECKSUM_NONE &&
2911 	    !can_checksum_protocol(features, type)) {
2912 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2913 	}
2914 	if (illegal_highdma(skb->dev, skb))
2915 		features &= ~NETIF_F_SG;
2916 
2917 	return features;
2918 }
2919 
passthru_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2920 netdev_features_t passthru_features_check(struct sk_buff *skb,
2921 					  struct net_device *dev,
2922 					  netdev_features_t features)
2923 {
2924 	return features;
2925 }
2926 EXPORT_SYMBOL(passthru_features_check);
2927 
dflt_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2928 static netdev_features_t dflt_features_check(struct sk_buff *skb,
2929 					     struct net_device *dev,
2930 					     netdev_features_t features)
2931 {
2932 	return vlan_features_check(skb, features);
2933 }
2934 
gso_features_check(const struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2935 static netdev_features_t gso_features_check(const struct sk_buff *skb,
2936 					    struct net_device *dev,
2937 					    netdev_features_t features)
2938 {
2939 	u16 gso_segs = skb_shinfo(skb)->gso_segs;
2940 
2941 	if (gso_segs > dev->gso_max_segs)
2942 		return features & ~NETIF_F_GSO_MASK;
2943 
2944 	/* Support for GSO partial features requires software
2945 	 * intervention before we can actually process the packets
2946 	 * so we need to strip support for any partial features now
2947 	 * and we can pull them back in after we have partially
2948 	 * segmented the frame.
2949 	 */
2950 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2951 		features &= ~dev->gso_partial_features;
2952 
2953 	/* Make sure to clear the IPv4 ID mangling feature if the
2954 	 * IPv4 header has the potential to be fragmented.
2955 	 */
2956 	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2957 		struct iphdr *iph = skb->encapsulation ?
2958 				    inner_ip_hdr(skb) : ip_hdr(skb);
2959 
2960 		if (!(iph->frag_off & htons(IP_DF)))
2961 			features &= ~NETIF_F_TSO_MANGLEID;
2962 	}
2963 
2964 	return features;
2965 }
2966 
netif_skb_features(struct sk_buff * skb)2967 netdev_features_t netif_skb_features(struct sk_buff *skb)
2968 {
2969 	struct net_device *dev = skb->dev;
2970 	netdev_features_t features = dev->features;
2971 
2972 	if (skb_is_gso(skb))
2973 		features = gso_features_check(skb, dev, features);
2974 
2975 	/* If encapsulation offload request, verify we are testing
2976 	 * hardware encapsulation features instead of standard
2977 	 * features for the netdev
2978 	 */
2979 	if (skb->encapsulation)
2980 		features &= dev->hw_enc_features;
2981 
2982 	if (skb_vlan_tagged(skb))
2983 		features = netdev_intersect_features(features,
2984 						     dev->vlan_features |
2985 						     NETIF_F_HW_VLAN_CTAG_TX |
2986 						     NETIF_F_HW_VLAN_STAG_TX);
2987 
2988 	if (dev->netdev_ops->ndo_features_check)
2989 		features &= dev->netdev_ops->ndo_features_check(skb, dev,
2990 								features);
2991 	else
2992 		features &= dflt_features_check(skb, dev, features);
2993 
2994 	return harmonize_features(skb, features);
2995 }
2996 EXPORT_SYMBOL(netif_skb_features);
2997 
xmit_one(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq,bool more)2998 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2999 		    struct netdev_queue *txq, bool more)
3000 {
3001 	unsigned int len;
3002 	int rc;
3003 
3004 	if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
3005 		dev_queue_xmit_nit(skb, dev);
3006 
3007 	len = skb->len;
3008 	trace_net_dev_start_xmit(skb, dev);
3009 	rc = netdev_start_xmit(skb, dev, txq, more);
3010 	trace_net_dev_xmit(skb, rc, dev, len);
3011 
3012 	return rc;
3013 }
3014 
dev_hard_start_xmit(struct sk_buff * first,struct net_device * dev,struct netdev_queue * txq,int * ret)3015 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3016 				    struct netdev_queue *txq, int *ret)
3017 {
3018 	struct sk_buff *skb = first;
3019 	int rc = NETDEV_TX_OK;
3020 
3021 	while (skb) {
3022 		struct sk_buff *next = skb->next;
3023 
3024 		skb->next = NULL;
3025 		rc = xmit_one(skb, dev, txq, next != NULL);
3026 		if (unlikely(!dev_xmit_complete(rc))) {
3027 			skb->next = next;
3028 			goto out;
3029 		}
3030 
3031 		skb = next;
3032 		if (netif_tx_queue_stopped(txq) && skb) {
3033 			rc = NETDEV_TX_BUSY;
3034 			break;
3035 		}
3036 	}
3037 
3038 out:
3039 	*ret = rc;
3040 	return skb;
3041 }
3042 
validate_xmit_vlan(struct sk_buff * skb,netdev_features_t features)3043 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3044 					  netdev_features_t features)
3045 {
3046 	if (skb_vlan_tag_present(skb) &&
3047 	    !vlan_hw_offload_capable(features, skb->vlan_proto))
3048 		skb = __vlan_hwaccel_push_inside(skb);
3049 	return skb;
3050 }
3051 
skb_csum_hwoffload_help(struct sk_buff * skb,const netdev_features_t features)3052 int skb_csum_hwoffload_help(struct sk_buff *skb,
3053 			    const netdev_features_t features)
3054 {
3055 	if (unlikely(skb->csum_not_inet))
3056 		return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3057 			skb_crc32c_csum_help(skb);
3058 
3059 	return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3060 }
3061 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3062 
validate_xmit_skb(struct sk_buff * skb,struct net_device * dev)3063 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
3064 {
3065 	netdev_features_t features;
3066 
3067 	features = netif_skb_features(skb);
3068 	skb = validate_xmit_vlan(skb, features);
3069 	if (unlikely(!skb))
3070 		goto out_null;
3071 
3072 	if (netif_needs_gso(skb, features)) {
3073 		struct sk_buff *segs;
3074 
3075 		segs = skb_gso_segment(skb, features);
3076 		if (IS_ERR(segs)) {
3077 			goto out_kfree_skb;
3078 		} else if (segs) {
3079 			consume_skb(skb);
3080 			skb = segs;
3081 		}
3082 	} else {
3083 		if (skb_needs_linearize(skb, features) &&
3084 		    __skb_linearize(skb))
3085 			goto out_kfree_skb;
3086 
3087 		if (validate_xmit_xfrm(skb, features))
3088 			goto out_kfree_skb;
3089 
3090 		/* If packet is not checksummed and device does not
3091 		 * support checksumming for this protocol, complete
3092 		 * checksumming here.
3093 		 */
3094 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
3095 			if (skb->encapsulation)
3096 				skb_set_inner_transport_header(skb,
3097 							       skb_checksum_start_offset(skb));
3098 			else
3099 				skb_set_transport_header(skb,
3100 							 skb_checksum_start_offset(skb));
3101 			if (skb_csum_hwoffload_help(skb, features))
3102 				goto out_kfree_skb;
3103 		}
3104 	}
3105 
3106 	return skb;
3107 
3108 out_kfree_skb:
3109 	kfree_skb(skb);
3110 out_null:
3111 	atomic_long_inc(&dev->tx_dropped);
3112 	return NULL;
3113 }
3114 
validate_xmit_skb_list(struct sk_buff * skb,struct net_device * dev)3115 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3116 {
3117 	struct sk_buff *next, *head = NULL, *tail;
3118 
3119 	for (; skb != NULL; skb = next) {
3120 		next = skb->next;
3121 		skb->next = NULL;
3122 
3123 		/* in case skb wont be segmented, point to itself */
3124 		skb->prev = skb;
3125 
3126 		skb = validate_xmit_skb(skb, dev);
3127 		if (!skb)
3128 			continue;
3129 
3130 		if (!head)
3131 			head = skb;
3132 		else
3133 			tail->next = skb;
3134 		/* If skb was segmented, skb->prev points to
3135 		 * the last segment. If not, it still contains skb.
3136 		 */
3137 		tail = skb->prev;
3138 	}
3139 	return head;
3140 }
3141 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3142 
qdisc_pkt_len_init(struct sk_buff * skb)3143 static void qdisc_pkt_len_init(struct sk_buff *skb)
3144 {
3145 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
3146 
3147 	qdisc_skb_cb(skb)->pkt_len = skb->len;
3148 
3149 	/* To get more precise estimation of bytes sent on wire,
3150 	 * we add to pkt_len the headers size of all segments
3151 	 */
3152 	if (shinfo->gso_size)  {
3153 		unsigned int hdr_len;
3154 		u16 gso_segs = shinfo->gso_segs;
3155 
3156 		/* mac layer + network layer */
3157 		hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3158 
3159 		/* + transport layer */
3160 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3161 			const struct tcphdr *th;
3162 			struct tcphdr _tcphdr;
3163 
3164 			th = skb_header_pointer(skb, skb_transport_offset(skb),
3165 						sizeof(_tcphdr), &_tcphdr);
3166 			if (likely(th))
3167 				hdr_len += __tcp_hdrlen(th);
3168 		} else {
3169 			struct udphdr _udphdr;
3170 
3171 			if (skb_header_pointer(skb, skb_transport_offset(skb),
3172 					       sizeof(_udphdr), &_udphdr))
3173 				hdr_len += sizeof(struct udphdr);
3174 		}
3175 
3176 		if (shinfo->gso_type & SKB_GSO_DODGY)
3177 			gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3178 						shinfo->gso_size);
3179 
3180 		qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3181 	}
3182 }
3183 
__dev_xmit_skb(struct sk_buff * skb,struct Qdisc * q,struct net_device * dev,struct netdev_queue * txq)3184 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3185 				 struct net_device *dev,
3186 				 struct netdev_queue *txq)
3187 {
3188 	spinlock_t *root_lock = qdisc_lock(q);
3189 	struct sk_buff *to_free = NULL;
3190 	bool contended;
3191 	int rc;
3192 
3193 	qdisc_calculate_pkt_len(skb, q);
3194 	/*
3195 	 * Heuristic to force contended enqueues to serialize on a
3196 	 * separate lock before trying to get qdisc main lock.
3197 	 * This permits qdisc->running owner to get the lock more
3198 	 * often and dequeue packets faster.
3199 	 */
3200 	contended = qdisc_is_running(q);
3201 	if (unlikely(contended))
3202 		spin_lock(&q->busylock);
3203 
3204 	spin_lock(root_lock);
3205 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3206 		__qdisc_drop(skb, &to_free);
3207 		rc = NET_XMIT_DROP;
3208 	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3209 		   qdisc_run_begin(q)) {
3210 		/*
3211 		 * This is a work-conserving queue; there are no old skbs
3212 		 * waiting to be sent out; and the qdisc is not running -
3213 		 * xmit the skb directly.
3214 		 */
3215 
3216 		qdisc_bstats_update(q, skb);
3217 
3218 		if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3219 			if (unlikely(contended)) {
3220 				spin_unlock(&q->busylock);
3221 				contended = false;
3222 			}
3223 			__qdisc_run(q);
3224 		} else
3225 			qdisc_run_end(q);
3226 
3227 		rc = NET_XMIT_SUCCESS;
3228 	} else {
3229 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3230 		if (qdisc_run_begin(q)) {
3231 			if (unlikely(contended)) {
3232 				spin_unlock(&q->busylock);
3233 				contended = false;
3234 			}
3235 			__qdisc_run(q);
3236 		}
3237 	}
3238 	spin_unlock(root_lock);
3239 	if (unlikely(to_free))
3240 		kfree_skb_list(to_free);
3241 	if (unlikely(contended))
3242 		spin_unlock(&q->busylock);
3243 	return rc;
3244 }
3245 
3246 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
skb_update_prio(struct sk_buff * skb)3247 static void skb_update_prio(struct sk_buff *skb)
3248 {
3249 	const struct netprio_map *map;
3250 	const struct sock *sk;
3251 	unsigned int prioidx;
3252 
3253 	if (skb->priority)
3254 		return;
3255 	map = rcu_dereference_bh(skb->dev->priomap);
3256 	if (!map)
3257 		return;
3258 	sk = skb_to_full_sk(skb);
3259 	if (!sk)
3260 		return;
3261 
3262 	prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3263 
3264 	if (prioidx < map->priomap_len)
3265 		skb->priority = map->priomap[prioidx];
3266 }
3267 #else
3268 #define skb_update_prio(skb)
3269 #endif
3270 
3271 DEFINE_PER_CPU(int, xmit_recursion);
3272 EXPORT_SYMBOL(xmit_recursion);
3273 
3274 /**
3275  *	dev_loopback_xmit - loop back @skb
3276  *	@net: network namespace this loopback is happening in
3277  *	@sk:  sk needed to be a netfilter okfn
3278  *	@skb: buffer to transmit
3279  */
dev_loopback_xmit(struct net * net,struct sock * sk,struct sk_buff * skb)3280 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3281 {
3282 	skb_reset_mac_header(skb);
3283 	__skb_pull(skb, skb_network_offset(skb));
3284 	skb->pkt_type = PACKET_LOOPBACK;
3285 	skb->ip_summed = CHECKSUM_UNNECESSARY;
3286 	WARN_ON(!skb_dst(skb));
3287 	skb_dst_force(skb);
3288 	netif_rx_ni(skb);
3289 	return 0;
3290 }
3291 EXPORT_SYMBOL(dev_loopback_xmit);
3292 
3293 #ifdef CONFIG_NET_EGRESS
3294 static struct sk_buff *
sch_handle_egress(struct sk_buff * skb,int * ret,struct net_device * dev)3295 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3296 {
3297 	struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3298 	struct tcf_result cl_res;
3299 
3300 	if (!cl)
3301 		return skb;
3302 
3303 	/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3304 	qdisc_bstats_cpu_update(cl->q, skb);
3305 
3306 	switch (tcf_classify(skb, cl, &cl_res, false)) {
3307 	case TC_ACT_OK:
3308 	case TC_ACT_RECLASSIFY:
3309 		skb->tc_index = TC_H_MIN(cl_res.classid);
3310 		break;
3311 	case TC_ACT_SHOT:
3312 		qdisc_qstats_cpu_drop(cl->q);
3313 		*ret = NET_XMIT_DROP;
3314 		kfree_skb(skb);
3315 		return NULL;
3316 	case TC_ACT_STOLEN:
3317 	case TC_ACT_QUEUED:
3318 	case TC_ACT_TRAP:
3319 		*ret = NET_XMIT_SUCCESS;
3320 		consume_skb(skb);
3321 		return NULL;
3322 	case TC_ACT_REDIRECT:
3323 		/* No need to push/pop skb's mac_header here on egress! */
3324 		skb_do_redirect(skb);
3325 		*ret = NET_XMIT_SUCCESS;
3326 		return NULL;
3327 	default:
3328 		break;
3329 	}
3330 
3331 	return skb;
3332 }
3333 #endif /* CONFIG_NET_EGRESS */
3334 
get_xps_queue(struct net_device * dev,struct sk_buff * skb)3335 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3336 {
3337 #ifdef CONFIG_XPS
3338 	struct xps_dev_maps *dev_maps;
3339 	struct xps_map *map;
3340 	int queue_index = -1;
3341 
3342 	rcu_read_lock();
3343 	dev_maps = rcu_dereference(dev->xps_maps);
3344 	if (dev_maps) {
3345 		unsigned int tci = skb->sender_cpu - 1;
3346 
3347 		if (dev->num_tc) {
3348 			tci *= dev->num_tc;
3349 			tci += netdev_get_prio_tc_map(dev, skb->priority);
3350 		}
3351 
3352 		map = rcu_dereference(dev_maps->cpu_map[tci]);
3353 		if (map) {
3354 			if (map->len == 1)
3355 				queue_index = map->queues[0];
3356 			else
3357 				queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3358 									   map->len)];
3359 			if (unlikely(queue_index >= dev->real_num_tx_queues))
3360 				queue_index = -1;
3361 		}
3362 	}
3363 	rcu_read_unlock();
3364 
3365 	return queue_index;
3366 #else
3367 	return -1;
3368 #endif
3369 }
3370 
__netdev_pick_tx(struct net_device * dev,struct sk_buff * skb)3371 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3372 {
3373 	struct sock *sk = skb->sk;
3374 	int queue_index = sk_tx_queue_get(sk);
3375 
3376 	if (queue_index < 0 || skb->ooo_okay ||
3377 	    queue_index >= dev->real_num_tx_queues) {
3378 		int new_index = get_xps_queue(dev, skb);
3379 
3380 		if (new_index < 0)
3381 			new_index = skb_tx_hash(dev, skb);
3382 
3383 		if (queue_index != new_index && sk &&
3384 		    sk_fullsock(sk) &&
3385 		    rcu_access_pointer(sk->sk_dst_cache))
3386 			sk_tx_queue_set(sk, new_index);
3387 
3388 		queue_index = new_index;
3389 	}
3390 
3391 	return queue_index;
3392 }
3393 
netdev_pick_tx(struct net_device * dev,struct sk_buff * skb,void * accel_priv)3394 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3395 				    struct sk_buff *skb,
3396 				    void *accel_priv)
3397 {
3398 	int queue_index = 0;
3399 
3400 #ifdef CONFIG_XPS
3401 	u32 sender_cpu = skb->sender_cpu - 1;
3402 
3403 	if (sender_cpu >= (u32)NR_CPUS)
3404 		skb->sender_cpu = raw_smp_processor_id() + 1;
3405 #endif
3406 
3407 	if (dev->real_num_tx_queues != 1) {
3408 		const struct net_device_ops *ops = dev->netdev_ops;
3409 
3410 		if (ops->ndo_select_queue)
3411 			queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3412 							    __netdev_pick_tx);
3413 		else
3414 			queue_index = __netdev_pick_tx(dev, skb);
3415 
3416 		if (!accel_priv)
3417 			queue_index = netdev_cap_txqueue(dev, queue_index);
3418 	}
3419 
3420 	skb_set_queue_mapping(skb, queue_index);
3421 	return netdev_get_tx_queue(dev, queue_index);
3422 }
3423 
3424 /**
3425  *	__dev_queue_xmit - transmit a buffer
3426  *	@skb: buffer to transmit
3427  *	@accel_priv: private data used for L2 forwarding offload
3428  *
3429  *	Queue a buffer for transmission to a network device. The caller must
3430  *	have set the device and priority and built the buffer before calling
3431  *	this function. The function can be called from an interrupt.
3432  *
3433  *	A negative errno code is returned on a failure. A success does not
3434  *	guarantee the frame will be transmitted as it may be dropped due
3435  *	to congestion or traffic shaping.
3436  *
3437  * -----------------------------------------------------------------------------------
3438  *      I notice this method can also return errors from the queue disciplines,
3439  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3440  *      be positive.
3441  *
3442  *      Regardless of the return value, the skb is consumed, so it is currently
3443  *      difficult to retry a send to this method.  (You can bump the ref count
3444  *      before sending to hold a reference for retry if you are careful.)
3445  *
3446  *      When calling this method, interrupts MUST be enabled.  This is because
3447  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3448  *          --BLG
3449  */
__dev_queue_xmit(struct sk_buff * skb,void * accel_priv)3450 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3451 {
3452 	struct net_device *dev = skb->dev;
3453 	struct netdev_queue *txq;
3454 	struct Qdisc *q;
3455 	int rc = -ENOMEM;
3456 
3457 	skb_reset_mac_header(skb);
3458 
3459 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3460 		__skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3461 
3462 	/* Disable soft irqs for various locks below. Also
3463 	 * stops preemption for RCU.
3464 	 */
3465 	rcu_read_lock_bh();
3466 
3467 	skb_update_prio(skb);
3468 
3469 	qdisc_pkt_len_init(skb);
3470 #ifdef CONFIG_NET_CLS_ACT
3471 	skb->tc_at_ingress = 0;
3472 # ifdef CONFIG_NET_EGRESS
3473 	if (static_key_false(&egress_needed)) {
3474 		skb = sch_handle_egress(skb, &rc, dev);
3475 		if (!skb)
3476 			goto out;
3477 	}
3478 # endif
3479 #endif
3480 	/* If device/qdisc don't need skb->dst, release it right now while
3481 	 * its hot in this cpu cache.
3482 	 */
3483 	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3484 		skb_dst_drop(skb);
3485 	else
3486 		skb_dst_force(skb);
3487 
3488 	txq = netdev_pick_tx(dev, skb, accel_priv);
3489 	q = rcu_dereference_bh(txq->qdisc);
3490 
3491 	trace_net_dev_queue(skb);
3492 	if (q->enqueue) {
3493 		rc = __dev_xmit_skb(skb, q, dev, txq);
3494 		goto out;
3495 	}
3496 
3497 	/* The device has no queue. Common case for software devices:
3498 	 * loopback, all the sorts of tunnels...
3499 
3500 	 * Really, it is unlikely that netif_tx_lock protection is necessary
3501 	 * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3502 	 * counters.)
3503 	 * However, it is possible, that they rely on protection
3504 	 * made by us here.
3505 
3506 	 * Check this and shot the lock. It is not prone from deadlocks.
3507 	 *Either shot noqueue qdisc, it is even simpler 8)
3508 	 */
3509 	if (dev->flags & IFF_UP) {
3510 		int cpu = smp_processor_id(); /* ok because BHs are off */
3511 
3512 		if (txq->xmit_lock_owner != cpu) {
3513 			if (unlikely(__this_cpu_read(xmit_recursion) >
3514 				     XMIT_RECURSION_LIMIT))
3515 				goto recursion_alert;
3516 
3517 			skb = validate_xmit_skb(skb, dev);
3518 			if (!skb)
3519 				goto out;
3520 
3521 			HARD_TX_LOCK(dev, txq, cpu);
3522 
3523 			if (!netif_xmit_stopped(txq)) {
3524 				__this_cpu_inc(xmit_recursion);
3525 				skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3526 				__this_cpu_dec(xmit_recursion);
3527 				if (dev_xmit_complete(rc)) {
3528 					HARD_TX_UNLOCK(dev, txq);
3529 					goto out;
3530 				}
3531 			}
3532 			HARD_TX_UNLOCK(dev, txq);
3533 			net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3534 					     dev->name);
3535 		} else {
3536 			/* Recursion is detected! It is possible,
3537 			 * unfortunately
3538 			 */
3539 recursion_alert:
3540 			net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3541 					     dev->name);
3542 		}
3543 	}
3544 
3545 	rc = -ENETDOWN;
3546 	rcu_read_unlock_bh();
3547 
3548 	atomic_long_inc(&dev->tx_dropped);
3549 	kfree_skb_list(skb);
3550 	return rc;
3551 out:
3552 	rcu_read_unlock_bh();
3553 	return rc;
3554 }
3555 
dev_queue_xmit(struct sk_buff * skb)3556 int dev_queue_xmit(struct sk_buff *skb)
3557 {
3558 	return __dev_queue_xmit(skb, NULL);
3559 }
3560 EXPORT_SYMBOL(dev_queue_xmit);
3561 
dev_queue_xmit_accel(struct sk_buff * skb,void * accel_priv)3562 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3563 {
3564 	return __dev_queue_xmit(skb, accel_priv);
3565 }
3566 EXPORT_SYMBOL(dev_queue_xmit_accel);
3567 
3568 
3569 /*************************************************************************
3570  *			Receiver routines
3571  *************************************************************************/
3572 
3573 int netdev_max_backlog __read_mostly = 1000;
3574 EXPORT_SYMBOL(netdev_max_backlog);
3575 
3576 int netdev_tstamp_prequeue __read_mostly = 1;
3577 int netdev_budget __read_mostly = 300;
3578 unsigned int __read_mostly netdev_budget_usecs = 2000;
3579 int weight_p __read_mostly = 64;           /* old backlog weight */
3580 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
3581 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
3582 int dev_rx_weight __read_mostly = 64;
3583 int dev_tx_weight __read_mostly = 64;
3584 
3585 /* Called with irq disabled */
____napi_schedule(struct softnet_data * sd,struct napi_struct * napi)3586 static inline void ____napi_schedule(struct softnet_data *sd,
3587 				     struct napi_struct *napi)
3588 {
3589 	list_add_tail(&napi->poll_list, &sd->poll_list);
3590 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3591 }
3592 
3593 #ifdef CONFIG_RPS
3594 
3595 /* One global table that all flow-based protocols share. */
3596 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3597 EXPORT_SYMBOL(rps_sock_flow_table);
3598 u32 rps_cpu_mask __read_mostly;
3599 EXPORT_SYMBOL(rps_cpu_mask);
3600 
3601 struct static_key rps_needed __read_mostly;
3602 EXPORT_SYMBOL(rps_needed);
3603 struct static_key rfs_needed __read_mostly;
3604 EXPORT_SYMBOL(rfs_needed);
3605 
3606 static struct rps_dev_flow *
set_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow * rflow,u16 next_cpu)3607 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3608 	    struct rps_dev_flow *rflow, u16 next_cpu)
3609 {
3610 	if (next_cpu < nr_cpu_ids) {
3611 #ifdef CONFIG_RFS_ACCEL
3612 		struct netdev_rx_queue *rxqueue;
3613 		struct rps_dev_flow_table *flow_table;
3614 		struct rps_dev_flow *old_rflow;
3615 		u32 flow_id;
3616 		u16 rxq_index;
3617 		int rc;
3618 
3619 		/* Should we steer this flow to a different hardware queue? */
3620 		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3621 		    !(dev->features & NETIF_F_NTUPLE))
3622 			goto out;
3623 		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3624 		if (rxq_index == skb_get_rx_queue(skb))
3625 			goto out;
3626 
3627 		rxqueue = dev->_rx + rxq_index;
3628 		flow_table = rcu_dereference(rxqueue->rps_flow_table);
3629 		if (!flow_table)
3630 			goto out;
3631 		flow_id = skb_get_hash(skb) & flow_table->mask;
3632 		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3633 							rxq_index, flow_id);
3634 		if (rc < 0)
3635 			goto out;
3636 		old_rflow = rflow;
3637 		rflow = &flow_table->flows[flow_id];
3638 		rflow->filter = rc;
3639 		if (old_rflow->filter == rflow->filter)
3640 			old_rflow->filter = RPS_NO_FILTER;
3641 	out:
3642 #endif
3643 		rflow->last_qtail =
3644 			per_cpu(softnet_data, next_cpu).input_queue_head;
3645 	}
3646 
3647 	rflow->cpu = next_cpu;
3648 	return rflow;
3649 }
3650 
3651 /*
3652  * get_rps_cpu is called from netif_receive_skb and returns the target
3653  * CPU from the RPS map of the receiving queue for a given skb.
3654  * rcu_read_lock must be held on entry.
3655  */
get_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow ** rflowp)3656 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3657 		       struct rps_dev_flow **rflowp)
3658 {
3659 	const struct rps_sock_flow_table *sock_flow_table;
3660 	struct netdev_rx_queue *rxqueue = dev->_rx;
3661 	struct rps_dev_flow_table *flow_table;
3662 	struct rps_map *map;
3663 	int cpu = -1;
3664 	u32 tcpu;
3665 	u32 hash;
3666 
3667 	if (skb_rx_queue_recorded(skb)) {
3668 		u16 index = skb_get_rx_queue(skb);
3669 
3670 		if (unlikely(index >= dev->real_num_rx_queues)) {
3671 			WARN_ONCE(dev->real_num_rx_queues > 1,
3672 				  "%s received packet on queue %u, but number "
3673 				  "of RX queues is %u\n",
3674 				  dev->name, index, dev->real_num_rx_queues);
3675 			goto done;
3676 		}
3677 		rxqueue += index;
3678 	}
3679 
3680 	/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3681 
3682 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3683 	map = rcu_dereference(rxqueue->rps_map);
3684 	if (!flow_table && !map)
3685 		goto done;
3686 
3687 	skb_reset_network_header(skb);
3688 	hash = skb_get_hash(skb);
3689 	if (!hash)
3690 		goto done;
3691 
3692 	sock_flow_table = rcu_dereference(rps_sock_flow_table);
3693 	if (flow_table && sock_flow_table) {
3694 		struct rps_dev_flow *rflow;
3695 		u32 next_cpu;
3696 		u32 ident;
3697 
3698 		/* First check into global flow table if there is a match */
3699 		ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3700 		if ((ident ^ hash) & ~rps_cpu_mask)
3701 			goto try_rps;
3702 
3703 		next_cpu = ident & rps_cpu_mask;
3704 
3705 		/* OK, now we know there is a match,
3706 		 * we can look at the local (per receive queue) flow table
3707 		 */
3708 		rflow = &flow_table->flows[hash & flow_table->mask];
3709 		tcpu = rflow->cpu;
3710 
3711 		/*
3712 		 * If the desired CPU (where last recvmsg was done) is
3713 		 * different from current CPU (one in the rx-queue flow
3714 		 * table entry), switch if one of the following holds:
3715 		 *   - Current CPU is unset (>= nr_cpu_ids).
3716 		 *   - Current CPU is offline.
3717 		 *   - The current CPU's queue tail has advanced beyond the
3718 		 *     last packet that was enqueued using this table entry.
3719 		 *     This guarantees that all previous packets for the flow
3720 		 *     have been dequeued, thus preserving in order delivery.
3721 		 */
3722 		if (unlikely(tcpu != next_cpu) &&
3723 		    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3724 		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3725 		      rflow->last_qtail)) >= 0)) {
3726 			tcpu = next_cpu;
3727 			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3728 		}
3729 
3730 		if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3731 			*rflowp = rflow;
3732 			cpu = tcpu;
3733 			goto done;
3734 		}
3735 	}
3736 
3737 try_rps:
3738 
3739 	if (map) {
3740 		tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3741 		if (cpu_online(tcpu)) {
3742 			cpu = tcpu;
3743 			goto done;
3744 		}
3745 	}
3746 
3747 done:
3748 	return cpu;
3749 }
3750 
3751 #ifdef CONFIG_RFS_ACCEL
3752 
3753 /**
3754  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3755  * @dev: Device on which the filter was set
3756  * @rxq_index: RX queue index
3757  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3758  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3759  *
3760  * Drivers that implement ndo_rx_flow_steer() should periodically call
3761  * this function for each installed filter and remove the filters for
3762  * which it returns %true.
3763  */
rps_may_expire_flow(struct net_device * dev,u16 rxq_index,u32 flow_id,u16 filter_id)3764 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3765 			 u32 flow_id, u16 filter_id)
3766 {
3767 	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3768 	struct rps_dev_flow_table *flow_table;
3769 	struct rps_dev_flow *rflow;
3770 	bool expire = true;
3771 	unsigned int cpu;
3772 
3773 	rcu_read_lock();
3774 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3775 	if (flow_table && flow_id <= flow_table->mask) {
3776 		rflow = &flow_table->flows[flow_id];
3777 		cpu = ACCESS_ONCE(rflow->cpu);
3778 		if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3779 		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3780 			   rflow->last_qtail) <
3781 		     (int)(10 * flow_table->mask)))
3782 			expire = false;
3783 	}
3784 	rcu_read_unlock();
3785 	return expire;
3786 }
3787 EXPORT_SYMBOL(rps_may_expire_flow);
3788 
3789 #endif /* CONFIG_RFS_ACCEL */
3790 
3791 /* Called from hardirq (IPI) context */
rps_trigger_softirq(void * data)3792 static void rps_trigger_softirq(void *data)
3793 {
3794 	struct softnet_data *sd = data;
3795 
3796 	____napi_schedule(sd, &sd->backlog);
3797 	sd->received_rps++;
3798 }
3799 
3800 #endif /* CONFIG_RPS */
3801 
3802 /*
3803  * Check if this softnet_data structure is another cpu one
3804  * If yes, queue it to our IPI list and return 1
3805  * If no, return 0
3806  */
rps_ipi_queued(struct softnet_data * sd)3807 static int rps_ipi_queued(struct softnet_data *sd)
3808 {
3809 #ifdef CONFIG_RPS
3810 	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3811 
3812 	if (sd != mysd) {
3813 		sd->rps_ipi_next = mysd->rps_ipi_list;
3814 		mysd->rps_ipi_list = sd;
3815 
3816 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3817 		return 1;
3818 	}
3819 #endif /* CONFIG_RPS */
3820 	return 0;
3821 }
3822 
3823 #ifdef CONFIG_NET_FLOW_LIMIT
3824 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3825 #endif
3826 
skb_flow_limit(struct sk_buff * skb,unsigned int qlen)3827 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3828 {
3829 #ifdef CONFIG_NET_FLOW_LIMIT
3830 	struct sd_flow_limit *fl;
3831 	struct softnet_data *sd;
3832 	unsigned int old_flow, new_flow;
3833 
3834 	if (qlen < (netdev_max_backlog >> 1))
3835 		return false;
3836 
3837 	sd = this_cpu_ptr(&softnet_data);
3838 
3839 	rcu_read_lock();
3840 	fl = rcu_dereference(sd->flow_limit);
3841 	if (fl) {
3842 		new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3843 		old_flow = fl->history[fl->history_head];
3844 		fl->history[fl->history_head] = new_flow;
3845 
3846 		fl->history_head++;
3847 		fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3848 
3849 		if (likely(fl->buckets[old_flow]))
3850 			fl->buckets[old_flow]--;
3851 
3852 		if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3853 			fl->count++;
3854 			rcu_read_unlock();
3855 			return true;
3856 		}
3857 	}
3858 	rcu_read_unlock();
3859 #endif
3860 	return false;
3861 }
3862 
3863 /*
3864  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3865  * queue (may be a remote CPU queue).
3866  */
enqueue_to_backlog(struct sk_buff * skb,int cpu,unsigned int * qtail)3867 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3868 			      unsigned int *qtail)
3869 {
3870 	struct softnet_data *sd;
3871 	unsigned long flags;
3872 	unsigned int qlen;
3873 
3874 	sd = &per_cpu(softnet_data, cpu);
3875 
3876 	local_irq_save(flags);
3877 
3878 	rps_lock(sd);
3879 	if (!netif_running(skb->dev))
3880 		goto drop;
3881 	qlen = skb_queue_len(&sd->input_pkt_queue);
3882 	if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3883 		if (qlen) {
3884 enqueue:
3885 			__skb_queue_tail(&sd->input_pkt_queue, skb);
3886 			input_queue_tail_incr_save(sd, qtail);
3887 			rps_unlock(sd);
3888 			local_irq_restore(flags);
3889 			return NET_RX_SUCCESS;
3890 		}
3891 
3892 		/* Schedule NAPI for backlog device
3893 		 * We can use non atomic operation since we own the queue lock
3894 		 */
3895 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3896 			if (!rps_ipi_queued(sd))
3897 				____napi_schedule(sd, &sd->backlog);
3898 		}
3899 		goto enqueue;
3900 	}
3901 
3902 drop:
3903 	sd->dropped++;
3904 	rps_unlock(sd);
3905 
3906 	local_irq_restore(flags);
3907 
3908 	atomic_long_inc(&skb->dev->rx_dropped);
3909 	kfree_skb(skb);
3910 	return NET_RX_DROP;
3911 }
3912 
netif_receive_generic_xdp(struct sk_buff * skb,struct bpf_prog * xdp_prog)3913 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
3914 				     struct bpf_prog *xdp_prog)
3915 {
3916 	struct xdp_buff xdp;
3917 	u32 act = XDP_DROP;
3918 	void *orig_data;
3919 	int hlen, off;
3920 	u32 mac_len;
3921 
3922 	/* Reinjected packets coming from act_mirred or similar should
3923 	 * not get XDP generic processing.
3924 	 */
3925 	if (skb_cloned(skb))
3926 		return XDP_PASS;
3927 
3928 	if (skb_linearize(skb))
3929 		goto do_drop;
3930 
3931 	/* The XDP program wants to see the packet starting at the MAC
3932 	 * header.
3933 	 */
3934 	mac_len = skb->data - skb_mac_header(skb);
3935 	hlen = skb_headlen(skb) + mac_len;
3936 	xdp.data = skb->data - mac_len;
3937 	xdp.data_end = xdp.data + hlen;
3938 	xdp.data_hard_start = skb->data - skb_headroom(skb);
3939 	orig_data = xdp.data;
3940 
3941 	act = bpf_prog_run_xdp(xdp_prog, &xdp);
3942 
3943 	off = xdp.data - orig_data;
3944 	if (off > 0)
3945 		__skb_pull(skb, off);
3946 	else if (off < 0)
3947 		__skb_push(skb, -off);
3948 	skb->mac_header += off;
3949 
3950 	switch (act) {
3951 	case XDP_REDIRECT:
3952 	case XDP_TX:
3953 		__skb_push(skb, mac_len);
3954 		/* fall through */
3955 	case XDP_PASS:
3956 		break;
3957 
3958 	default:
3959 		bpf_warn_invalid_xdp_action(act);
3960 		/* fall through */
3961 	case XDP_ABORTED:
3962 		trace_xdp_exception(skb->dev, xdp_prog, act);
3963 		/* fall through */
3964 	case XDP_DROP:
3965 	do_drop:
3966 		kfree_skb(skb);
3967 		break;
3968 	}
3969 
3970 	return act;
3971 }
3972 
3973 /* When doing generic XDP we have to bypass the qdisc layer and the
3974  * network taps in order to match in-driver-XDP behavior.
3975  */
generic_xdp_tx(struct sk_buff * skb,struct bpf_prog * xdp_prog)3976 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
3977 {
3978 	struct net_device *dev = skb->dev;
3979 	struct netdev_queue *txq;
3980 	bool free_skb = true;
3981 	int cpu, rc;
3982 
3983 	txq = netdev_pick_tx(dev, skb, NULL);
3984 	cpu = smp_processor_id();
3985 	HARD_TX_LOCK(dev, txq, cpu);
3986 	if (!netif_xmit_stopped(txq)) {
3987 		rc = netdev_start_xmit(skb, dev, txq, 0);
3988 		if (dev_xmit_complete(rc))
3989 			free_skb = false;
3990 	}
3991 	HARD_TX_UNLOCK(dev, txq);
3992 	if (free_skb) {
3993 		trace_xdp_exception(dev, xdp_prog, XDP_TX);
3994 		kfree_skb(skb);
3995 	}
3996 }
3997 EXPORT_SYMBOL_GPL(generic_xdp_tx);
3998 
3999 static struct static_key generic_xdp_needed __read_mostly;
4000 
do_xdp_generic(struct bpf_prog * xdp_prog,struct sk_buff * skb)4001 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4002 {
4003 	if (xdp_prog) {
4004 		u32 act = netif_receive_generic_xdp(skb, xdp_prog);
4005 		int err;
4006 
4007 		if (act != XDP_PASS) {
4008 			switch (act) {
4009 			case XDP_REDIRECT:
4010 				err = xdp_do_generic_redirect(skb->dev, skb,
4011 							      xdp_prog);
4012 				if (err)
4013 					goto out_redir;
4014 			/* fallthru to submit skb */
4015 			case XDP_TX:
4016 				generic_xdp_tx(skb, xdp_prog);
4017 				break;
4018 			}
4019 			return XDP_DROP;
4020 		}
4021 	}
4022 	return XDP_PASS;
4023 out_redir:
4024 	kfree_skb(skb);
4025 	return XDP_DROP;
4026 }
4027 EXPORT_SYMBOL_GPL(do_xdp_generic);
4028 
netif_rx_internal(struct sk_buff * skb)4029 static int netif_rx_internal(struct sk_buff *skb)
4030 {
4031 	int ret;
4032 
4033 	net_timestamp_check(netdev_tstamp_prequeue, skb);
4034 
4035 	trace_netif_rx(skb);
4036 
4037 	if (static_key_false(&generic_xdp_needed)) {
4038 		int ret;
4039 
4040 		preempt_disable();
4041 		rcu_read_lock();
4042 		ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4043 		rcu_read_unlock();
4044 		preempt_enable();
4045 
4046 		/* Consider XDP consuming the packet a success from
4047 		 * the netdev point of view we do not want to count
4048 		 * this as an error.
4049 		 */
4050 		if (ret != XDP_PASS)
4051 			return NET_RX_SUCCESS;
4052 	}
4053 
4054 #ifdef CONFIG_RPS
4055 	if (static_key_false(&rps_needed)) {
4056 		struct rps_dev_flow voidflow, *rflow = &voidflow;
4057 		int cpu;
4058 
4059 		preempt_disable();
4060 		rcu_read_lock();
4061 
4062 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
4063 		if (cpu < 0)
4064 			cpu = smp_processor_id();
4065 
4066 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4067 
4068 		rcu_read_unlock();
4069 		preempt_enable();
4070 	} else
4071 #endif
4072 	{
4073 		unsigned int qtail;
4074 
4075 		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4076 		put_cpu();
4077 	}
4078 	return ret;
4079 }
4080 
4081 /**
4082  *	netif_rx	-	post buffer to the network code
4083  *	@skb: buffer to post
4084  *
4085  *	This function receives a packet from a device driver and queues it for
4086  *	the upper (protocol) levels to process.  It always succeeds. The buffer
4087  *	may be dropped during processing for congestion control or by the
4088  *	protocol layers.
4089  *
4090  *	return values:
4091  *	NET_RX_SUCCESS	(no congestion)
4092  *	NET_RX_DROP     (packet was dropped)
4093  *
4094  */
4095 
netif_rx(struct sk_buff * skb)4096 int netif_rx(struct sk_buff *skb)
4097 {
4098 	trace_netif_rx_entry(skb);
4099 
4100 	return netif_rx_internal(skb);
4101 }
4102 EXPORT_SYMBOL(netif_rx);
4103 
netif_rx_ni(struct sk_buff * skb)4104 int netif_rx_ni(struct sk_buff *skb)
4105 {
4106 	int err;
4107 
4108 	trace_netif_rx_ni_entry(skb);
4109 
4110 	preempt_disable();
4111 	err = netif_rx_internal(skb);
4112 	if (local_softirq_pending())
4113 		do_softirq();
4114 	preempt_enable();
4115 
4116 	return err;
4117 }
4118 EXPORT_SYMBOL(netif_rx_ni);
4119 
net_tx_action(struct softirq_action * h)4120 static __latent_entropy void net_tx_action(struct softirq_action *h)
4121 {
4122 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4123 
4124 	if (sd->completion_queue) {
4125 		struct sk_buff *clist;
4126 
4127 		local_irq_disable();
4128 		clist = sd->completion_queue;
4129 		sd->completion_queue = NULL;
4130 		local_irq_enable();
4131 
4132 		while (clist) {
4133 			struct sk_buff *skb = clist;
4134 
4135 			clist = clist->next;
4136 
4137 			WARN_ON(refcount_read(&skb->users));
4138 			if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4139 				trace_consume_skb(skb);
4140 			else
4141 				trace_kfree_skb(skb, net_tx_action);
4142 
4143 			if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4144 				__kfree_skb(skb);
4145 			else
4146 				__kfree_skb_defer(skb);
4147 		}
4148 
4149 		__kfree_skb_flush();
4150 	}
4151 
4152 	if (sd->output_queue) {
4153 		struct Qdisc *head;
4154 
4155 		local_irq_disable();
4156 		head = sd->output_queue;
4157 		sd->output_queue = NULL;
4158 		sd->output_queue_tailp = &sd->output_queue;
4159 		local_irq_enable();
4160 
4161 		while (head) {
4162 			struct Qdisc *q = head;
4163 			spinlock_t *root_lock;
4164 
4165 			head = head->next_sched;
4166 
4167 			root_lock = qdisc_lock(q);
4168 			spin_lock(root_lock);
4169 			/* We need to make sure head->next_sched is read
4170 			 * before clearing __QDISC_STATE_SCHED
4171 			 */
4172 			smp_mb__before_atomic();
4173 			clear_bit(__QDISC_STATE_SCHED, &q->state);
4174 			qdisc_run(q);
4175 			spin_unlock(root_lock);
4176 		}
4177 	}
4178 }
4179 
4180 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4181 /* This hook is defined here for ATM LANE */
4182 int (*br_fdb_test_addr_hook)(struct net_device *dev,
4183 			     unsigned char *addr) __read_mostly;
4184 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
4185 #endif
4186 
4187 static inline struct sk_buff *
sch_handle_ingress(struct sk_buff * skb,struct packet_type ** pt_prev,int * ret,struct net_device * orig_dev)4188 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4189 		   struct net_device *orig_dev)
4190 {
4191 #ifdef CONFIG_NET_CLS_ACT
4192 	struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
4193 	struct tcf_result cl_res;
4194 
4195 	/* If there's at least one ingress present somewhere (so
4196 	 * we get here via enabled static key), remaining devices
4197 	 * that are not configured with an ingress qdisc will bail
4198 	 * out here.
4199 	 */
4200 	if (!cl)
4201 		return skb;
4202 	if (*pt_prev) {
4203 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
4204 		*pt_prev = NULL;
4205 	}
4206 
4207 	qdisc_skb_cb(skb)->pkt_len = skb->len;
4208 	skb->tc_at_ingress = 1;
4209 	qdisc_bstats_cpu_update(cl->q, skb);
4210 
4211 	switch (tcf_classify(skb, cl, &cl_res, false)) {
4212 	case TC_ACT_OK:
4213 	case TC_ACT_RECLASSIFY:
4214 		skb->tc_index = TC_H_MIN(cl_res.classid);
4215 		break;
4216 	case TC_ACT_SHOT:
4217 		qdisc_qstats_cpu_drop(cl->q);
4218 		kfree_skb(skb);
4219 		return NULL;
4220 	case TC_ACT_STOLEN:
4221 	case TC_ACT_QUEUED:
4222 	case TC_ACT_TRAP:
4223 		consume_skb(skb);
4224 		return NULL;
4225 	case TC_ACT_REDIRECT:
4226 		/* skb_mac_header check was done by cls/act_bpf, so
4227 		 * we can safely push the L2 header back before
4228 		 * redirecting to another netdev
4229 		 */
4230 		__skb_push(skb, skb->mac_len);
4231 		skb_do_redirect(skb);
4232 		return NULL;
4233 	default:
4234 		break;
4235 	}
4236 #endif /* CONFIG_NET_CLS_ACT */
4237 	return skb;
4238 }
4239 
4240 /**
4241  *	netdev_is_rx_handler_busy - check if receive handler is registered
4242  *	@dev: device to check
4243  *
4244  *	Check if a receive handler is already registered for a given device.
4245  *	Return true if there one.
4246  *
4247  *	The caller must hold the rtnl_mutex.
4248  */
netdev_is_rx_handler_busy(struct net_device * dev)4249 bool netdev_is_rx_handler_busy(struct net_device *dev)
4250 {
4251 	ASSERT_RTNL();
4252 	return dev && rtnl_dereference(dev->rx_handler);
4253 }
4254 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4255 
4256 /**
4257  *	netdev_rx_handler_register - register receive handler
4258  *	@dev: device to register a handler for
4259  *	@rx_handler: receive handler to register
4260  *	@rx_handler_data: data pointer that is used by rx handler
4261  *
4262  *	Register a receive handler for a device. This handler will then be
4263  *	called from __netif_receive_skb. A negative errno code is returned
4264  *	on a failure.
4265  *
4266  *	The caller must hold the rtnl_mutex.
4267  *
4268  *	For a general description of rx_handler, see enum rx_handler_result.
4269  */
netdev_rx_handler_register(struct net_device * dev,rx_handler_func_t * rx_handler,void * rx_handler_data)4270 int netdev_rx_handler_register(struct net_device *dev,
4271 			       rx_handler_func_t *rx_handler,
4272 			       void *rx_handler_data)
4273 {
4274 	if (netdev_is_rx_handler_busy(dev))
4275 		return -EBUSY;
4276 
4277 	/* Note: rx_handler_data must be set before rx_handler */
4278 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4279 	rcu_assign_pointer(dev->rx_handler, rx_handler);
4280 
4281 	return 0;
4282 }
4283 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4284 
4285 /**
4286  *	netdev_rx_handler_unregister - unregister receive handler
4287  *	@dev: device to unregister a handler from
4288  *
4289  *	Unregister a receive handler from a device.
4290  *
4291  *	The caller must hold the rtnl_mutex.
4292  */
netdev_rx_handler_unregister(struct net_device * dev)4293 void netdev_rx_handler_unregister(struct net_device *dev)
4294 {
4295 
4296 	ASSERT_RTNL();
4297 	RCU_INIT_POINTER(dev->rx_handler, NULL);
4298 	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4299 	 * section has a guarantee to see a non NULL rx_handler_data
4300 	 * as well.
4301 	 */
4302 	synchronize_net();
4303 	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4304 }
4305 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4306 
4307 /*
4308  * Limit the use of PFMEMALLOC reserves to those protocols that implement
4309  * the special handling of PFMEMALLOC skbs.
4310  */
skb_pfmemalloc_protocol(struct sk_buff * skb)4311 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4312 {
4313 	switch (skb->protocol) {
4314 	case htons(ETH_P_ARP):
4315 	case htons(ETH_P_IP):
4316 	case htons(ETH_P_IPV6):
4317 	case htons(ETH_P_8021Q):
4318 	case htons(ETH_P_8021AD):
4319 		return true;
4320 	default:
4321 		return false;
4322 	}
4323 }
4324 
nf_ingress(struct sk_buff * skb,struct packet_type ** pt_prev,int * ret,struct net_device * orig_dev)4325 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4326 			     int *ret, struct net_device *orig_dev)
4327 {
4328 #ifdef CONFIG_NETFILTER_INGRESS
4329 	if (nf_hook_ingress_active(skb)) {
4330 		int ingress_retval;
4331 
4332 		if (*pt_prev) {
4333 			*ret = deliver_skb(skb, *pt_prev, orig_dev);
4334 			*pt_prev = NULL;
4335 		}
4336 
4337 		rcu_read_lock();
4338 		ingress_retval = nf_hook_ingress(skb);
4339 		rcu_read_unlock();
4340 		return ingress_retval;
4341 	}
4342 #endif /* CONFIG_NETFILTER_INGRESS */
4343 	return 0;
4344 }
4345 
__netif_receive_skb_core(struct sk_buff * skb,bool pfmemalloc)4346 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4347 {
4348 	struct packet_type *ptype, *pt_prev;
4349 	rx_handler_func_t *rx_handler;
4350 	struct net_device *orig_dev;
4351 	bool deliver_exact = false;
4352 	int ret = NET_RX_DROP;
4353 	__be16 type;
4354 
4355 	net_timestamp_check(!netdev_tstamp_prequeue, skb);
4356 
4357 	trace_netif_receive_skb(skb);
4358 
4359 	orig_dev = skb->dev;
4360 
4361 	skb_reset_network_header(skb);
4362 	if (!skb_transport_header_was_set(skb))
4363 		skb_reset_transport_header(skb);
4364 	skb_reset_mac_len(skb);
4365 
4366 	pt_prev = NULL;
4367 
4368 another_round:
4369 	skb->skb_iif = skb->dev->ifindex;
4370 
4371 	__this_cpu_inc(softnet_data.processed);
4372 
4373 	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4374 	    skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4375 		skb = skb_vlan_untag(skb);
4376 		if (unlikely(!skb))
4377 			goto out;
4378 	}
4379 
4380 	if (skb_skip_tc_classify(skb))
4381 		goto skip_classify;
4382 
4383 	if (pfmemalloc)
4384 		goto skip_taps;
4385 
4386 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
4387 		if (pt_prev)
4388 			ret = deliver_skb(skb, pt_prev, orig_dev);
4389 		pt_prev = ptype;
4390 	}
4391 
4392 	list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4393 		if (pt_prev)
4394 			ret = deliver_skb(skb, pt_prev, orig_dev);
4395 		pt_prev = ptype;
4396 	}
4397 
4398 skip_taps:
4399 #ifdef CONFIG_NET_INGRESS
4400 	if (static_key_false(&ingress_needed)) {
4401 		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4402 		if (!skb)
4403 			goto out;
4404 
4405 		if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4406 			goto out;
4407 	}
4408 #endif
4409 	skb_reset_tc(skb);
4410 skip_classify:
4411 	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4412 		goto drop;
4413 
4414 	if (skb_vlan_tag_present(skb)) {
4415 		if (pt_prev) {
4416 			ret = deliver_skb(skb, pt_prev, orig_dev);
4417 			pt_prev = NULL;
4418 		}
4419 		if (vlan_do_receive(&skb))
4420 			goto another_round;
4421 		else if (unlikely(!skb))
4422 			goto out;
4423 	}
4424 
4425 	rx_handler = rcu_dereference(skb->dev->rx_handler);
4426 	if (rx_handler) {
4427 		if (pt_prev) {
4428 			ret = deliver_skb(skb, pt_prev, orig_dev);
4429 			pt_prev = NULL;
4430 		}
4431 		switch (rx_handler(&skb)) {
4432 		case RX_HANDLER_CONSUMED:
4433 			ret = NET_RX_SUCCESS;
4434 			goto out;
4435 		case RX_HANDLER_ANOTHER:
4436 			goto another_round;
4437 		case RX_HANDLER_EXACT:
4438 			deliver_exact = true;
4439 		case RX_HANDLER_PASS:
4440 			break;
4441 		default:
4442 			BUG();
4443 		}
4444 	}
4445 
4446 	if (unlikely(skb_vlan_tag_present(skb))) {
4447 		if (skb_vlan_tag_get_id(skb))
4448 			skb->pkt_type = PACKET_OTHERHOST;
4449 		/* Note: we might in the future use prio bits
4450 		 * and set skb->priority like in vlan_do_receive()
4451 		 * For the time being, just ignore Priority Code Point
4452 		 */
4453 		skb->vlan_tci = 0;
4454 	}
4455 
4456 	type = skb->protocol;
4457 
4458 	/* deliver only exact match when indicated */
4459 	if (likely(!deliver_exact)) {
4460 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4461 				       &ptype_base[ntohs(type) &
4462 						   PTYPE_HASH_MASK]);
4463 	}
4464 
4465 	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4466 			       &orig_dev->ptype_specific);
4467 
4468 	if (unlikely(skb->dev != orig_dev)) {
4469 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4470 				       &skb->dev->ptype_specific);
4471 	}
4472 
4473 	if (pt_prev) {
4474 		if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
4475 			goto drop;
4476 		else
4477 			ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4478 	} else {
4479 drop:
4480 		if (!deliver_exact)
4481 			atomic_long_inc(&skb->dev->rx_dropped);
4482 		else
4483 			atomic_long_inc(&skb->dev->rx_nohandler);
4484 		kfree_skb(skb);
4485 		/* Jamal, now you will not able to escape explaining
4486 		 * me how you were going to use this. :-)
4487 		 */
4488 		ret = NET_RX_DROP;
4489 	}
4490 
4491 out:
4492 	return ret;
4493 }
4494 
__netif_receive_skb(struct sk_buff * skb)4495 static int __netif_receive_skb(struct sk_buff *skb)
4496 {
4497 	int ret;
4498 
4499 	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4500 		unsigned int noreclaim_flag;
4501 
4502 		/*
4503 		 * PFMEMALLOC skbs are special, they should
4504 		 * - be delivered to SOCK_MEMALLOC sockets only
4505 		 * - stay away from userspace
4506 		 * - have bounded memory usage
4507 		 *
4508 		 * Use PF_MEMALLOC as this saves us from propagating the allocation
4509 		 * context down to all allocation sites.
4510 		 */
4511 		noreclaim_flag = memalloc_noreclaim_save();
4512 		ret = __netif_receive_skb_core(skb, true);
4513 		memalloc_noreclaim_restore(noreclaim_flag);
4514 	} else
4515 		ret = __netif_receive_skb_core(skb, false);
4516 
4517 	return ret;
4518 }
4519 
generic_xdp_install(struct net_device * dev,struct netdev_xdp * xdp)4520 static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
4521 {
4522 	struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
4523 	struct bpf_prog *new = xdp->prog;
4524 	int ret = 0;
4525 
4526 	switch (xdp->command) {
4527 	case XDP_SETUP_PROG:
4528 		rcu_assign_pointer(dev->xdp_prog, new);
4529 		if (old)
4530 			bpf_prog_put(old);
4531 
4532 		if (old && !new) {
4533 			static_key_slow_dec(&generic_xdp_needed);
4534 		} else if (new && !old) {
4535 			static_key_slow_inc(&generic_xdp_needed);
4536 			dev_disable_lro(dev);
4537 		}
4538 		break;
4539 
4540 	case XDP_QUERY_PROG:
4541 		xdp->prog_attached = !!old;
4542 		xdp->prog_id = old ? old->aux->id : 0;
4543 		break;
4544 
4545 	default:
4546 		ret = -EINVAL;
4547 		break;
4548 	}
4549 
4550 	return ret;
4551 }
4552 
netif_receive_skb_internal(struct sk_buff * skb)4553 static int netif_receive_skb_internal(struct sk_buff *skb)
4554 {
4555 	int ret;
4556 
4557 	net_timestamp_check(netdev_tstamp_prequeue, skb);
4558 
4559 	if (skb_defer_rx_timestamp(skb))
4560 		return NET_RX_SUCCESS;
4561 
4562 	if (static_key_false(&generic_xdp_needed)) {
4563 		int ret;
4564 
4565 		preempt_disable();
4566 		rcu_read_lock();
4567 		ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4568 		rcu_read_unlock();
4569 		preempt_enable();
4570 
4571 		if (ret != XDP_PASS)
4572 			return NET_RX_DROP;
4573 	}
4574 
4575 	rcu_read_lock();
4576 #ifdef CONFIG_RPS
4577 	if (static_key_false(&rps_needed)) {
4578 		struct rps_dev_flow voidflow, *rflow = &voidflow;
4579 		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4580 
4581 		if (cpu >= 0) {
4582 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4583 			rcu_read_unlock();
4584 			return ret;
4585 		}
4586 	}
4587 #endif
4588 	ret = __netif_receive_skb(skb);
4589 	rcu_read_unlock();
4590 	return ret;
4591 }
4592 
4593 /**
4594  *	netif_receive_skb - process receive buffer from network
4595  *	@skb: buffer to process
4596  *
4597  *	netif_receive_skb() is the main receive data processing function.
4598  *	It always succeeds. The buffer may be dropped during processing
4599  *	for congestion control or by the protocol layers.
4600  *
4601  *	This function may only be called from softirq context and interrupts
4602  *	should be enabled.
4603  *
4604  *	Return values (usually ignored):
4605  *	NET_RX_SUCCESS: no congestion
4606  *	NET_RX_DROP: packet was dropped
4607  */
netif_receive_skb(struct sk_buff * skb)4608 int netif_receive_skb(struct sk_buff *skb)
4609 {
4610 	trace_netif_receive_skb_entry(skb);
4611 
4612 	return netif_receive_skb_internal(skb);
4613 }
4614 EXPORT_SYMBOL(netif_receive_skb);
4615 
4616 DEFINE_PER_CPU(struct work_struct, flush_works);
4617 
4618 /* Network device is going away, flush any packets still pending */
flush_backlog(struct work_struct * work)4619 static void flush_backlog(struct work_struct *work)
4620 {
4621 	struct sk_buff *skb, *tmp;
4622 	struct softnet_data *sd;
4623 
4624 	local_bh_disable();
4625 	sd = this_cpu_ptr(&softnet_data);
4626 
4627 	local_irq_disable();
4628 	rps_lock(sd);
4629 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4630 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4631 			__skb_unlink(skb, &sd->input_pkt_queue);
4632 			kfree_skb(skb);
4633 			input_queue_head_incr(sd);
4634 		}
4635 	}
4636 	rps_unlock(sd);
4637 	local_irq_enable();
4638 
4639 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4640 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4641 			__skb_unlink(skb, &sd->process_queue);
4642 			kfree_skb(skb);
4643 			input_queue_head_incr(sd);
4644 		}
4645 	}
4646 	local_bh_enable();
4647 }
4648 
flush_all_backlogs(void)4649 static void flush_all_backlogs(void)
4650 {
4651 	unsigned int cpu;
4652 
4653 	get_online_cpus();
4654 
4655 	for_each_online_cpu(cpu)
4656 		queue_work_on(cpu, system_highpri_wq,
4657 			      per_cpu_ptr(&flush_works, cpu));
4658 
4659 	for_each_online_cpu(cpu)
4660 		flush_work(per_cpu_ptr(&flush_works, cpu));
4661 
4662 	put_online_cpus();
4663 }
4664 
napi_gro_complete(struct sk_buff * skb)4665 static int napi_gro_complete(struct sk_buff *skb)
4666 {
4667 	struct packet_offload *ptype;
4668 	__be16 type = skb->protocol;
4669 	struct list_head *head = &offload_base;
4670 	int err = -ENOENT;
4671 
4672 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4673 
4674 	if (NAPI_GRO_CB(skb)->count == 1) {
4675 		skb_shinfo(skb)->gso_size = 0;
4676 		goto out;
4677 	}
4678 
4679 	rcu_read_lock();
4680 	list_for_each_entry_rcu(ptype, head, list) {
4681 		if (ptype->type != type || !ptype->callbacks.gro_complete)
4682 			continue;
4683 
4684 		err = ptype->callbacks.gro_complete(skb, 0);
4685 		break;
4686 	}
4687 	rcu_read_unlock();
4688 
4689 	if (err) {
4690 		WARN_ON(&ptype->list == head);
4691 		kfree_skb(skb);
4692 		return NET_RX_SUCCESS;
4693 	}
4694 
4695 out:
4696 	return netif_receive_skb_internal(skb);
4697 }
4698 
4699 /* napi->gro_list contains packets ordered by age.
4700  * youngest packets at the head of it.
4701  * Complete skbs in reverse order to reduce latencies.
4702  */
napi_gro_flush(struct napi_struct * napi,bool flush_old)4703 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4704 {
4705 	struct sk_buff *skb, *prev = NULL;
4706 
4707 	/* scan list and build reverse chain */
4708 	for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4709 		skb->prev = prev;
4710 		prev = skb;
4711 	}
4712 
4713 	for (skb = prev; skb; skb = prev) {
4714 		skb->next = NULL;
4715 
4716 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4717 			return;
4718 
4719 		prev = skb->prev;
4720 		napi_gro_complete(skb);
4721 		napi->gro_count--;
4722 	}
4723 
4724 	napi->gro_list = NULL;
4725 }
4726 EXPORT_SYMBOL(napi_gro_flush);
4727 
gro_list_prepare(struct napi_struct * napi,struct sk_buff * skb)4728 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4729 {
4730 	struct sk_buff *p;
4731 	unsigned int maclen = skb->dev->hard_header_len;
4732 	u32 hash = skb_get_hash_raw(skb);
4733 
4734 	for (p = napi->gro_list; p; p = p->next) {
4735 		unsigned long diffs;
4736 
4737 		NAPI_GRO_CB(p)->flush = 0;
4738 
4739 		if (hash != skb_get_hash_raw(p)) {
4740 			NAPI_GRO_CB(p)->same_flow = 0;
4741 			continue;
4742 		}
4743 
4744 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4745 		diffs |= p->vlan_tci ^ skb->vlan_tci;
4746 		diffs |= skb_metadata_dst_cmp(p, skb);
4747 		if (maclen == ETH_HLEN)
4748 			diffs |= compare_ether_header(skb_mac_header(p),
4749 						      skb_mac_header(skb));
4750 		else if (!diffs)
4751 			diffs = memcmp(skb_mac_header(p),
4752 				       skb_mac_header(skb),
4753 				       maclen);
4754 		NAPI_GRO_CB(p)->same_flow = !diffs;
4755 	}
4756 }
4757 
skb_gro_reset_offset(struct sk_buff * skb)4758 static void skb_gro_reset_offset(struct sk_buff *skb)
4759 {
4760 	const struct skb_shared_info *pinfo = skb_shinfo(skb);
4761 	const skb_frag_t *frag0 = &pinfo->frags[0];
4762 
4763 	NAPI_GRO_CB(skb)->data_offset = 0;
4764 	NAPI_GRO_CB(skb)->frag0 = NULL;
4765 	NAPI_GRO_CB(skb)->frag0_len = 0;
4766 
4767 	if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4768 	    pinfo->nr_frags &&
4769 	    !PageHighMem(skb_frag_page(frag0))) {
4770 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4771 		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4772 						    skb_frag_size(frag0),
4773 						    skb->end - skb->tail);
4774 	}
4775 }
4776 
gro_pull_from_frag0(struct sk_buff * skb,int grow)4777 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4778 {
4779 	struct skb_shared_info *pinfo = skb_shinfo(skb);
4780 
4781 	BUG_ON(skb->end - skb->tail < grow);
4782 
4783 	memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4784 
4785 	skb->data_len -= grow;
4786 	skb->tail += grow;
4787 
4788 	pinfo->frags[0].page_offset += grow;
4789 	skb_frag_size_sub(&pinfo->frags[0], grow);
4790 
4791 	if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4792 		skb_frag_unref(skb, 0);
4793 		memmove(pinfo->frags, pinfo->frags + 1,
4794 			--pinfo->nr_frags * sizeof(pinfo->frags[0]));
4795 	}
4796 }
4797 
dev_gro_receive(struct napi_struct * napi,struct sk_buff * skb)4798 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4799 {
4800 	struct sk_buff **pp = NULL;
4801 	struct packet_offload *ptype;
4802 	__be16 type = skb->protocol;
4803 	struct list_head *head = &offload_base;
4804 	int same_flow;
4805 	enum gro_result ret;
4806 	int grow;
4807 
4808 	if (netif_elide_gro(skb->dev))
4809 		goto normal;
4810 
4811 	gro_list_prepare(napi, skb);
4812 
4813 	rcu_read_lock();
4814 	list_for_each_entry_rcu(ptype, head, list) {
4815 		if (ptype->type != type || !ptype->callbacks.gro_receive)
4816 			continue;
4817 
4818 		skb_set_network_header(skb, skb_gro_offset(skb));
4819 		skb_reset_mac_len(skb);
4820 		NAPI_GRO_CB(skb)->same_flow = 0;
4821 		NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
4822 		NAPI_GRO_CB(skb)->free = 0;
4823 		NAPI_GRO_CB(skb)->encap_mark = 0;
4824 		NAPI_GRO_CB(skb)->recursion_counter = 0;
4825 		NAPI_GRO_CB(skb)->is_fou = 0;
4826 		NAPI_GRO_CB(skb)->is_atomic = 1;
4827 		NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4828 
4829 		/* Setup for GRO checksum validation */
4830 		switch (skb->ip_summed) {
4831 		case CHECKSUM_COMPLETE:
4832 			NAPI_GRO_CB(skb)->csum = skb->csum;
4833 			NAPI_GRO_CB(skb)->csum_valid = 1;
4834 			NAPI_GRO_CB(skb)->csum_cnt = 0;
4835 			break;
4836 		case CHECKSUM_UNNECESSARY:
4837 			NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4838 			NAPI_GRO_CB(skb)->csum_valid = 0;
4839 			break;
4840 		default:
4841 			NAPI_GRO_CB(skb)->csum_cnt = 0;
4842 			NAPI_GRO_CB(skb)->csum_valid = 0;
4843 		}
4844 
4845 		pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4846 		break;
4847 	}
4848 	rcu_read_unlock();
4849 
4850 	if (&ptype->list == head)
4851 		goto normal;
4852 
4853 	if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4854 		ret = GRO_CONSUMED;
4855 		goto ok;
4856 	}
4857 
4858 	same_flow = NAPI_GRO_CB(skb)->same_flow;
4859 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4860 
4861 	if (pp) {
4862 		struct sk_buff *nskb = *pp;
4863 
4864 		*pp = nskb->next;
4865 		nskb->next = NULL;
4866 		napi_gro_complete(nskb);
4867 		napi->gro_count--;
4868 	}
4869 
4870 	if (same_flow)
4871 		goto ok;
4872 
4873 	if (NAPI_GRO_CB(skb)->flush)
4874 		goto normal;
4875 
4876 	if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4877 		struct sk_buff *nskb = napi->gro_list;
4878 
4879 		/* locate the end of the list to select the 'oldest' flow */
4880 		while (nskb->next) {
4881 			pp = &nskb->next;
4882 			nskb = *pp;
4883 		}
4884 		*pp = NULL;
4885 		nskb->next = NULL;
4886 		napi_gro_complete(nskb);
4887 	} else {
4888 		napi->gro_count++;
4889 	}
4890 	NAPI_GRO_CB(skb)->count = 1;
4891 	NAPI_GRO_CB(skb)->age = jiffies;
4892 	NAPI_GRO_CB(skb)->last = skb;
4893 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4894 	skb->next = napi->gro_list;
4895 	napi->gro_list = skb;
4896 	ret = GRO_HELD;
4897 
4898 pull:
4899 	grow = skb_gro_offset(skb) - skb_headlen(skb);
4900 	if (grow > 0)
4901 		gro_pull_from_frag0(skb, grow);
4902 ok:
4903 	return ret;
4904 
4905 normal:
4906 	ret = GRO_NORMAL;
4907 	goto pull;
4908 }
4909 
gro_find_receive_by_type(__be16 type)4910 struct packet_offload *gro_find_receive_by_type(__be16 type)
4911 {
4912 	struct list_head *offload_head = &offload_base;
4913 	struct packet_offload *ptype;
4914 
4915 	list_for_each_entry_rcu(ptype, offload_head, list) {
4916 		if (ptype->type != type || !ptype->callbacks.gro_receive)
4917 			continue;
4918 		return ptype;
4919 	}
4920 	return NULL;
4921 }
4922 EXPORT_SYMBOL(gro_find_receive_by_type);
4923 
gro_find_complete_by_type(__be16 type)4924 struct packet_offload *gro_find_complete_by_type(__be16 type)
4925 {
4926 	struct list_head *offload_head = &offload_base;
4927 	struct packet_offload *ptype;
4928 
4929 	list_for_each_entry_rcu(ptype, offload_head, list) {
4930 		if (ptype->type != type || !ptype->callbacks.gro_complete)
4931 			continue;
4932 		return ptype;
4933 	}
4934 	return NULL;
4935 }
4936 EXPORT_SYMBOL(gro_find_complete_by_type);
4937 
napi_skb_free_stolen_head(struct sk_buff * skb)4938 static void napi_skb_free_stolen_head(struct sk_buff *skb)
4939 {
4940 	skb_dst_drop(skb);
4941 	secpath_reset(skb);
4942 	kmem_cache_free(skbuff_head_cache, skb);
4943 }
4944 
napi_skb_finish(gro_result_t ret,struct sk_buff * skb)4945 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4946 {
4947 	switch (ret) {
4948 	case GRO_NORMAL:
4949 		if (netif_receive_skb_internal(skb))
4950 			ret = GRO_DROP;
4951 		break;
4952 
4953 	case GRO_DROP:
4954 		kfree_skb(skb);
4955 		break;
4956 
4957 	case GRO_MERGED_FREE:
4958 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4959 			napi_skb_free_stolen_head(skb);
4960 		else
4961 			__kfree_skb(skb);
4962 		break;
4963 
4964 	case GRO_HELD:
4965 	case GRO_MERGED:
4966 	case GRO_CONSUMED:
4967 		break;
4968 	}
4969 
4970 	return ret;
4971 }
4972 
napi_gro_receive(struct napi_struct * napi,struct sk_buff * skb)4973 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4974 {
4975 	skb_mark_napi_id(skb, napi);
4976 	trace_napi_gro_receive_entry(skb);
4977 
4978 	skb_gro_reset_offset(skb);
4979 
4980 	return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4981 }
4982 EXPORT_SYMBOL(napi_gro_receive);
4983 
napi_reuse_skb(struct napi_struct * napi,struct sk_buff * skb)4984 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4985 {
4986 	if (unlikely(skb->pfmemalloc)) {
4987 		consume_skb(skb);
4988 		return;
4989 	}
4990 	__skb_pull(skb, skb_headlen(skb));
4991 	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
4992 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4993 	skb->vlan_tci = 0;
4994 	skb->dev = napi->dev;
4995 	skb->skb_iif = 0;
4996 
4997 	/* eth_type_trans() assumes pkt_type is PACKET_HOST */
4998 	skb->pkt_type = PACKET_HOST;
4999 
5000 	skb->encapsulation = 0;
5001 	skb_shinfo(skb)->gso_type = 0;
5002 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5003 	secpath_reset(skb);
5004 
5005 	napi->skb = skb;
5006 }
5007 
napi_get_frags(struct napi_struct * napi)5008 struct sk_buff *napi_get_frags(struct napi_struct *napi)
5009 {
5010 	struct sk_buff *skb = napi->skb;
5011 
5012 	if (!skb) {
5013 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
5014 		if (skb) {
5015 			napi->skb = skb;
5016 			skb_mark_napi_id(skb, napi);
5017 		}
5018 	}
5019 	return skb;
5020 }
5021 EXPORT_SYMBOL(napi_get_frags);
5022 
napi_frags_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)5023 static gro_result_t napi_frags_finish(struct napi_struct *napi,
5024 				      struct sk_buff *skb,
5025 				      gro_result_t ret)
5026 {
5027 	switch (ret) {
5028 	case GRO_NORMAL:
5029 	case GRO_HELD:
5030 		__skb_push(skb, ETH_HLEN);
5031 		skb->protocol = eth_type_trans(skb, skb->dev);
5032 		if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
5033 			ret = GRO_DROP;
5034 		break;
5035 
5036 	case GRO_DROP:
5037 		napi_reuse_skb(napi, skb);
5038 		break;
5039 
5040 	case GRO_MERGED_FREE:
5041 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5042 			napi_skb_free_stolen_head(skb);
5043 		else
5044 			napi_reuse_skb(napi, skb);
5045 		break;
5046 
5047 	case GRO_MERGED:
5048 	case GRO_CONSUMED:
5049 		break;
5050 	}
5051 
5052 	return ret;
5053 }
5054 
5055 /* Upper GRO stack assumes network header starts at gro_offset=0
5056  * Drivers could call both napi_gro_frags() and napi_gro_receive()
5057  * We copy ethernet header into skb->data to have a common layout.
5058  */
napi_frags_skb(struct napi_struct * napi)5059 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
5060 {
5061 	struct sk_buff *skb = napi->skb;
5062 	const struct ethhdr *eth;
5063 	unsigned int hlen = sizeof(*eth);
5064 
5065 	napi->skb = NULL;
5066 
5067 	skb_reset_mac_header(skb);
5068 	skb_gro_reset_offset(skb);
5069 
5070 	if (unlikely(skb_gro_header_hard(skb, hlen))) {
5071 		eth = skb_gro_header_slow(skb, hlen, 0);
5072 		if (unlikely(!eth)) {
5073 			net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5074 					     __func__, napi->dev->name);
5075 			napi_reuse_skb(napi, skb);
5076 			return NULL;
5077 		}
5078 	} else {
5079 		eth = (const struct ethhdr *)skb->data;
5080 		gro_pull_from_frag0(skb, hlen);
5081 		NAPI_GRO_CB(skb)->frag0 += hlen;
5082 		NAPI_GRO_CB(skb)->frag0_len -= hlen;
5083 	}
5084 	__skb_pull(skb, hlen);
5085 
5086 	/*
5087 	 * This works because the only protocols we care about don't require
5088 	 * special handling.
5089 	 * We'll fix it up properly in napi_frags_finish()
5090 	 */
5091 	skb->protocol = eth->h_proto;
5092 
5093 	return skb;
5094 }
5095 
napi_gro_frags(struct napi_struct * napi)5096 gro_result_t napi_gro_frags(struct napi_struct *napi)
5097 {
5098 	struct sk_buff *skb = napi_frags_skb(napi);
5099 
5100 	if (!skb)
5101 		return GRO_DROP;
5102 
5103 	trace_napi_gro_frags_entry(skb);
5104 
5105 	return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5106 }
5107 EXPORT_SYMBOL(napi_gro_frags);
5108 
5109 /* Compute the checksum from gro_offset and return the folded value
5110  * after adding in any pseudo checksum.
5111  */
__skb_gro_checksum_complete(struct sk_buff * skb)5112 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5113 {
5114 	__wsum wsum;
5115 	__sum16 sum;
5116 
5117 	wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5118 
5119 	/* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5120 	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5121 	if (likely(!sum)) {
5122 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5123 		    !skb->csum_complete_sw)
5124 			netdev_rx_csum_fault(skb->dev);
5125 	}
5126 
5127 	NAPI_GRO_CB(skb)->csum = wsum;
5128 	NAPI_GRO_CB(skb)->csum_valid = 1;
5129 
5130 	return sum;
5131 }
5132 EXPORT_SYMBOL(__skb_gro_checksum_complete);
5133 
net_rps_send_ipi(struct softnet_data * remsd)5134 static void net_rps_send_ipi(struct softnet_data *remsd)
5135 {
5136 #ifdef CONFIG_RPS
5137 	while (remsd) {
5138 		struct softnet_data *next = remsd->rps_ipi_next;
5139 
5140 		if (cpu_online(remsd->cpu))
5141 			smp_call_function_single_async(remsd->cpu, &remsd->csd);
5142 		remsd = next;
5143 	}
5144 #endif
5145 }
5146 
5147 /*
5148  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5149  * Note: called with local irq disabled, but exits with local irq enabled.
5150  */
net_rps_action_and_irq_enable(struct softnet_data * sd)5151 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5152 {
5153 #ifdef CONFIG_RPS
5154 	struct softnet_data *remsd = sd->rps_ipi_list;
5155 
5156 	if (remsd) {
5157 		sd->rps_ipi_list = NULL;
5158 
5159 		local_irq_enable();
5160 
5161 		/* Send pending IPI's to kick RPS processing on remote cpus. */
5162 		net_rps_send_ipi(remsd);
5163 	} else
5164 #endif
5165 		local_irq_enable();
5166 }
5167 
sd_has_rps_ipi_waiting(struct softnet_data * sd)5168 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5169 {
5170 #ifdef CONFIG_RPS
5171 	return sd->rps_ipi_list != NULL;
5172 #else
5173 	return false;
5174 #endif
5175 }
5176 
process_backlog(struct napi_struct * napi,int quota)5177 static int process_backlog(struct napi_struct *napi, int quota)
5178 {
5179 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5180 	bool again = true;
5181 	int work = 0;
5182 
5183 	/* Check if we have pending ipi, its better to send them now,
5184 	 * not waiting net_rx_action() end.
5185 	 */
5186 	if (sd_has_rps_ipi_waiting(sd)) {
5187 		local_irq_disable();
5188 		net_rps_action_and_irq_enable(sd);
5189 	}
5190 
5191 	napi->weight = dev_rx_weight;
5192 	while (again) {
5193 		struct sk_buff *skb;
5194 
5195 		while ((skb = __skb_dequeue(&sd->process_queue))) {
5196 			rcu_read_lock();
5197 			__netif_receive_skb(skb);
5198 			rcu_read_unlock();
5199 			input_queue_head_incr(sd);
5200 			if (++work >= quota)
5201 				return work;
5202 
5203 		}
5204 
5205 		local_irq_disable();
5206 		rps_lock(sd);
5207 		if (skb_queue_empty(&sd->input_pkt_queue)) {
5208 			/*
5209 			 * Inline a custom version of __napi_complete().
5210 			 * only current cpu owns and manipulates this napi,
5211 			 * and NAPI_STATE_SCHED is the only possible flag set
5212 			 * on backlog.
5213 			 * We can use a plain write instead of clear_bit(),
5214 			 * and we dont need an smp_mb() memory barrier.
5215 			 */
5216 			napi->state = 0;
5217 			again = false;
5218 		} else {
5219 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
5220 						   &sd->process_queue);
5221 		}
5222 		rps_unlock(sd);
5223 		local_irq_enable();
5224 	}
5225 
5226 	return work;
5227 }
5228 
5229 /**
5230  * __napi_schedule - schedule for receive
5231  * @n: entry to schedule
5232  *
5233  * The entry's receive function will be scheduled to run.
5234  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5235  */
__napi_schedule(struct napi_struct * n)5236 void __napi_schedule(struct napi_struct *n)
5237 {
5238 	unsigned long flags;
5239 
5240 	local_irq_save(flags);
5241 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
5242 	local_irq_restore(flags);
5243 }
5244 EXPORT_SYMBOL(__napi_schedule);
5245 
5246 /**
5247  *	napi_schedule_prep - check if napi can be scheduled
5248  *	@n: napi context
5249  *
5250  * Test if NAPI routine is already running, and if not mark
5251  * it as running.  This is used as a condition variable
5252  * insure only one NAPI poll instance runs.  We also make
5253  * sure there is no pending NAPI disable.
5254  */
napi_schedule_prep(struct napi_struct * n)5255 bool napi_schedule_prep(struct napi_struct *n)
5256 {
5257 	unsigned long val, new;
5258 
5259 	do {
5260 		val = READ_ONCE(n->state);
5261 		if (unlikely(val & NAPIF_STATE_DISABLE))
5262 			return false;
5263 		new = val | NAPIF_STATE_SCHED;
5264 
5265 		/* Sets STATE_MISSED bit if STATE_SCHED was already set
5266 		 * This was suggested by Alexander Duyck, as compiler
5267 		 * emits better code than :
5268 		 * if (val & NAPIF_STATE_SCHED)
5269 		 *     new |= NAPIF_STATE_MISSED;
5270 		 */
5271 		new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5272 						   NAPIF_STATE_MISSED;
5273 	} while (cmpxchg(&n->state, val, new) != val);
5274 
5275 	return !(val & NAPIF_STATE_SCHED);
5276 }
5277 EXPORT_SYMBOL(napi_schedule_prep);
5278 
5279 /**
5280  * __napi_schedule_irqoff - schedule for receive
5281  * @n: entry to schedule
5282  *
5283  * Variant of __napi_schedule() assuming hard irqs are masked
5284  */
__napi_schedule_irqoff(struct napi_struct * n)5285 void __napi_schedule_irqoff(struct napi_struct *n)
5286 {
5287 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
5288 }
5289 EXPORT_SYMBOL(__napi_schedule_irqoff);
5290 
napi_complete_done(struct napi_struct * n,int work_done)5291 bool napi_complete_done(struct napi_struct *n, int work_done)
5292 {
5293 	unsigned long flags, val, new;
5294 
5295 	/*
5296 	 * 1) Don't let napi dequeue from the cpu poll list
5297 	 *    just in case its running on a different cpu.
5298 	 * 2) If we are busy polling, do nothing here, we have
5299 	 *    the guarantee we will be called later.
5300 	 */
5301 	if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5302 				 NAPIF_STATE_IN_BUSY_POLL)))
5303 		return false;
5304 
5305 	if (n->gro_list) {
5306 		unsigned long timeout = 0;
5307 
5308 		if (work_done)
5309 			timeout = n->dev->gro_flush_timeout;
5310 
5311 		/* When the NAPI instance uses a timeout and keeps postponing
5312 		 * it, we need to bound somehow the time packets are kept in
5313 		 * the GRO layer
5314 		 */
5315 		napi_gro_flush(n, !!timeout);
5316 		if (timeout)
5317 			hrtimer_start(&n->timer, ns_to_ktime(timeout),
5318 				      HRTIMER_MODE_REL_PINNED);
5319 	}
5320 	if (unlikely(!list_empty(&n->poll_list))) {
5321 		/* If n->poll_list is not empty, we need to mask irqs */
5322 		local_irq_save(flags);
5323 		list_del_init(&n->poll_list);
5324 		local_irq_restore(flags);
5325 	}
5326 
5327 	do {
5328 		val = READ_ONCE(n->state);
5329 
5330 		WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5331 
5332 		new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5333 
5334 		/* If STATE_MISSED was set, leave STATE_SCHED set,
5335 		 * because we will call napi->poll() one more time.
5336 		 * This C code was suggested by Alexander Duyck to help gcc.
5337 		 */
5338 		new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5339 						    NAPIF_STATE_SCHED;
5340 	} while (cmpxchg(&n->state, val, new) != val);
5341 
5342 	if (unlikely(val & NAPIF_STATE_MISSED)) {
5343 		__napi_schedule(n);
5344 		return false;
5345 	}
5346 
5347 	return true;
5348 }
5349 EXPORT_SYMBOL(napi_complete_done);
5350 
5351 /* must be called under rcu_read_lock(), as we dont take a reference */
napi_by_id(unsigned int napi_id)5352 static struct napi_struct *napi_by_id(unsigned int napi_id)
5353 {
5354 	unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5355 	struct napi_struct *napi;
5356 
5357 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5358 		if (napi->napi_id == napi_id)
5359 			return napi;
5360 
5361 	return NULL;
5362 }
5363 
5364 #if defined(CONFIG_NET_RX_BUSY_POLL)
5365 
5366 #define BUSY_POLL_BUDGET 8
5367 
busy_poll_stop(struct napi_struct * napi,void * have_poll_lock)5368 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5369 {
5370 	int rc;
5371 
5372 	/* Busy polling means there is a high chance device driver hard irq
5373 	 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
5374 	 * set in napi_schedule_prep().
5375 	 * Since we are about to call napi->poll() once more, we can safely
5376 	 * clear NAPI_STATE_MISSED.
5377 	 *
5378 	 * Note: x86 could use a single "lock and ..." instruction
5379 	 * to perform these two clear_bit()
5380 	 */
5381 	clear_bit(NAPI_STATE_MISSED, &napi->state);
5382 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5383 
5384 	local_bh_disable();
5385 
5386 	/* All we really want here is to re-enable device interrupts.
5387 	 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
5388 	 */
5389 	rc = napi->poll(napi, BUSY_POLL_BUDGET);
5390 	trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5391 	netpoll_poll_unlock(have_poll_lock);
5392 	if (rc == BUSY_POLL_BUDGET)
5393 		__napi_schedule(napi);
5394 	local_bh_enable();
5395 }
5396 
napi_busy_loop(unsigned int napi_id,bool (* loop_end)(void *,unsigned long),void * loop_end_arg)5397 void napi_busy_loop(unsigned int napi_id,
5398 		    bool (*loop_end)(void *, unsigned long),
5399 		    void *loop_end_arg)
5400 {
5401 	unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
5402 	int (*napi_poll)(struct napi_struct *napi, int budget);
5403 	void *have_poll_lock = NULL;
5404 	struct napi_struct *napi;
5405 
5406 restart:
5407 	napi_poll = NULL;
5408 
5409 	rcu_read_lock();
5410 
5411 	napi = napi_by_id(napi_id);
5412 	if (!napi)
5413 		goto out;
5414 
5415 	preempt_disable();
5416 	for (;;) {
5417 		int work = 0;
5418 
5419 		local_bh_disable();
5420 		if (!napi_poll) {
5421 			unsigned long val = READ_ONCE(napi->state);
5422 
5423 			/* If multiple threads are competing for this napi,
5424 			 * we avoid dirtying napi->state as much as we can.
5425 			 */
5426 			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5427 				   NAPIF_STATE_IN_BUSY_POLL))
5428 				goto count;
5429 			if (cmpxchg(&napi->state, val,
5430 				    val | NAPIF_STATE_IN_BUSY_POLL |
5431 					  NAPIF_STATE_SCHED) != val)
5432 				goto count;
5433 			have_poll_lock = netpoll_poll_lock(napi);
5434 			napi_poll = napi->poll;
5435 		}
5436 		work = napi_poll(napi, BUSY_POLL_BUDGET);
5437 		trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
5438 count:
5439 		if (work > 0)
5440 			__NET_ADD_STATS(dev_net(napi->dev),
5441 					LINUX_MIB_BUSYPOLLRXPACKETS, work);
5442 		local_bh_enable();
5443 
5444 		if (!loop_end || loop_end(loop_end_arg, start_time))
5445 			break;
5446 
5447 		if (unlikely(need_resched())) {
5448 			if (napi_poll)
5449 				busy_poll_stop(napi, have_poll_lock);
5450 			preempt_enable();
5451 			rcu_read_unlock();
5452 			cond_resched();
5453 			if (loop_end(loop_end_arg, start_time))
5454 				return;
5455 			goto restart;
5456 		}
5457 		cpu_relax();
5458 	}
5459 	if (napi_poll)
5460 		busy_poll_stop(napi, have_poll_lock);
5461 	preempt_enable();
5462 out:
5463 	rcu_read_unlock();
5464 }
5465 EXPORT_SYMBOL(napi_busy_loop);
5466 
5467 #endif /* CONFIG_NET_RX_BUSY_POLL */
5468 
napi_hash_add(struct napi_struct * napi)5469 static void napi_hash_add(struct napi_struct *napi)
5470 {
5471 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5472 	    test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5473 		return;
5474 
5475 	spin_lock(&napi_hash_lock);
5476 
5477 	/* 0..NR_CPUS range is reserved for sender_cpu use */
5478 	do {
5479 		if (unlikely(++napi_gen_id < MIN_NAPI_ID))
5480 			napi_gen_id = MIN_NAPI_ID;
5481 	} while (napi_by_id(napi_gen_id));
5482 	napi->napi_id = napi_gen_id;
5483 
5484 	hlist_add_head_rcu(&napi->napi_hash_node,
5485 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5486 
5487 	spin_unlock(&napi_hash_lock);
5488 }
5489 
5490 /* Warning : caller is responsible to make sure rcu grace period
5491  * is respected before freeing memory containing @napi
5492  */
napi_hash_del(struct napi_struct * napi)5493 bool napi_hash_del(struct napi_struct *napi)
5494 {
5495 	bool rcu_sync_needed = false;
5496 
5497 	spin_lock(&napi_hash_lock);
5498 
5499 	if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5500 		rcu_sync_needed = true;
5501 		hlist_del_rcu(&napi->napi_hash_node);
5502 	}
5503 	spin_unlock(&napi_hash_lock);
5504 	return rcu_sync_needed;
5505 }
5506 EXPORT_SYMBOL_GPL(napi_hash_del);
5507 
napi_watchdog(struct hrtimer * timer)5508 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5509 {
5510 	struct napi_struct *napi;
5511 
5512 	napi = container_of(timer, struct napi_struct, timer);
5513 
5514 	/* Note : we use a relaxed variant of napi_schedule_prep() not setting
5515 	 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
5516 	 */
5517 	if (napi->gro_list && !napi_disable_pending(napi) &&
5518 	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5519 		__napi_schedule_irqoff(napi);
5520 
5521 	return HRTIMER_NORESTART;
5522 }
5523 
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)5524 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5525 		    int (*poll)(struct napi_struct *, int), int weight)
5526 {
5527 	INIT_LIST_HEAD(&napi->poll_list);
5528 	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5529 	napi->timer.function = napi_watchdog;
5530 	napi->gro_count = 0;
5531 	napi->gro_list = NULL;
5532 	napi->skb = NULL;
5533 	napi->poll = poll;
5534 	if (weight > NAPI_POLL_WEIGHT)
5535 		pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5536 			    weight, dev->name);
5537 	napi->weight = weight;
5538 	list_add(&napi->dev_list, &dev->napi_list);
5539 	napi->dev = dev;
5540 #ifdef CONFIG_NETPOLL
5541 	napi->poll_owner = -1;
5542 #endif
5543 	set_bit(NAPI_STATE_SCHED, &napi->state);
5544 	napi_hash_add(napi);
5545 }
5546 EXPORT_SYMBOL(netif_napi_add);
5547 
napi_disable(struct napi_struct * n)5548 void napi_disable(struct napi_struct *n)
5549 {
5550 	might_sleep();
5551 	set_bit(NAPI_STATE_DISABLE, &n->state);
5552 
5553 	while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5554 		msleep(1);
5555 	while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5556 		msleep(1);
5557 
5558 	hrtimer_cancel(&n->timer);
5559 
5560 	clear_bit(NAPI_STATE_DISABLE, &n->state);
5561 }
5562 EXPORT_SYMBOL(napi_disable);
5563 
5564 /* Must be called in process context */
netif_napi_del(struct napi_struct * napi)5565 void netif_napi_del(struct napi_struct *napi)
5566 {
5567 	might_sleep();
5568 	if (napi_hash_del(napi))
5569 		synchronize_net();
5570 	list_del_init(&napi->dev_list);
5571 	napi_free_frags(napi);
5572 
5573 	kfree_skb_list(napi->gro_list);
5574 	napi->gro_list = NULL;
5575 	napi->gro_count = 0;
5576 }
5577 EXPORT_SYMBOL(netif_napi_del);
5578 
napi_poll(struct napi_struct * n,struct list_head * repoll)5579 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5580 {
5581 	void *have;
5582 	int work, weight;
5583 
5584 	list_del_init(&n->poll_list);
5585 
5586 	have = netpoll_poll_lock(n);
5587 
5588 	weight = n->weight;
5589 
5590 	/* This NAPI_STATE_SCHED test is for avoiding a race
5591 	 * with netpoll's poll_napi().  Only the entity which
5592 	 * obtains the lock and sees NAPI_STATE_SCHED set will
5593 	 * actually make the ->poll() call.  Therefore we avoid
5594 	 * accidentally calling ->poll() when NAPI is not scheduled.
5595 	 */
5596 	work = 0;
5597 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5598 		work = n->poll(n, weight);
5599 		trace_napi_poll(n, work, weight);
5600 	}
5601 
5602 	WARN_ON_ONCE(work > weight);
5603 
5604 	if (likely(work < weight))
5605 		goto out_unlock;
5606 
5607 	/* Drivers must not modify the NAPI state if they
5608 	 * consume the entire weight.  In such cases this code
5609 	 * still "owns" the NAPI instance and therefore can
5610 	 * move the instance around on the list at-will.
5611 	 */
5612 	if (unlikely(napi_disable_pending(n))) {
5613 		napi_complete(n);
5614 		goto out_unlock;
5615 	}
5616 
5617 	if (n->gro_list) {
5618 		/* flush too old packets
5619 		 * If HZ < 1000, flush all packets.
5620 		 */
5621 		napi_gro_flush(n, HZ >= 1000);
5622 	}
5623 
5624 	/* Some drivers may have called napi_schedule
5625 	 * prior to exhausting their budget.
5626 	 */
5627 	if (unlikely(!list_empty(&n->poll_list))) {
5628 		pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5629 			     n->dev ? n->dev->name : "backlog");
5630 		goto out_unlock;
5631 	}
5632 
5633 	list_add_tail(&n->poll_list, repoll);
5634 
5635 out_unlock:
5636 	netpoll_poll_unlock(have);
5637 
5638 	return work;
5639 }
5640 
net_rx_action(struct softirq_action * h)5641 static __latent_entropy void net_rx_action(struct softirq_action *h)
5642 {
5643 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5644 	unsigned long time_limit = jiffies +
5645 		usecs_to_jiffies(netdev_budget_usecs);
5646 	int budget = netdev_budget;
5647 	LIST_HEAD(list);
5648 	LIST_HEAD(repoll);
5649 
5650 	local_irq_disable();
5651 	list_splice_init(&sd->poll_list, &list);
5652 	local_irq_enable();
5653 
5654 	for (;;) {
5655 		struct napi_struct *n;
5656 
5657 		if (list_empty(&list)) {
5658 			if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5659 				goto out;
5660 			break;
5661 		}
5662 
5663 		n = list_first_entry(&list, struct napi_struct, poll_list);
5664 		budget -= napi_poll(n, &repoll);
5665 
5666 		/* If softirq window is exhausted then punt.
5667 		 * Allow this to run for 2 jiffies since which will allow
5668 		 * an average latency of 1.5/HZ.
5669 		 */
5670 		if (unlikely(budget <= 0 ||
5671 			     time_after_eq(jiffies, time_limit))) {
5672 			sd->time_squeeze++;
5673 			break;
5674 		}
5675 	}
5676 
5677 	local_irq_disable();
5678 
5679 	list_splice_tail_init(&sd->poll_list, &list);
5680 	list_splice_tail(&repoll, &list);
5681 	list_splice(&list, &sd->poll_list);
5682 	if (!list_empty(&sd->poll_list))
5683 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5684 
5685 	net_rps_action_and_irq_enable(sd);
5686 out:
5687 	__kfree_skb_flush();
5688 }
5689 
5690 struct netdev_adjacent {
5691 	struct net_device *dev;
5692 
5693 	/* upper master flag, there can only be one master device per list */
5694 	bool master;
5695 
5696 	/* counter for the number of times this device was added to us */
5697 	u16 ref_nr;
5698 
5699 	/* private field for the users */
5700 	void *private;
5701 
5702 	struct list_head list;
5703 	struct rcu_head rcu;
5704 };
5705 
__netdev_find_adj(struct net_device * adj_dev,struct list_head * adj_list)5706 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5707 						 struct list_head *adj_list)
5708 {
5709 	struct netdev_adjacent *adj;
5710 
5711 	list_for_each_entry(adj, adj_list, list) {
5712 		if (adj->dev == adj_dev)
5713 			return adj;
5714 	}
5715 	return NULL;
5716 }
5717 
__netdev_has_upper_dev(struct net_device * upper_dev,void * data)5718 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5719 {
5720 	struct net_device *dev = data;
5721 
5722 	return upper_dev == dev;
5723 }
5724 
5725 /**
5726  * netdev_has_upper_dev - Check if device is linked to an upper device
5727  * @dev: device
5728  * @upper_dev: upper device to check
5729  *
5730  * Find out if a device is linked to specified upper device and return true
5731  * in case it is. Note that this checks only immediate upper device,
5732  * not through a complete stack of devices. The caller must hold the RTNL lock.
5733  */
netdev_has_upper_dev(struct net_device * dev,struct net_device * upper_dev)5734 bool netdev_has_upper_dev(struct net_device *dev,
5735 			  struct net_device *upper_dev)
5736 {
5737 	ASSERT_RTNL();
5738 
5739 	return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5740 					     upper_dev);
5741 }
5742 EXPORT_SYMBOL(netdev_has_upper_dev);
5743 
5744 /**
5745  * netdev_has_upper_dev_all - Check if device is linked to an upper device
5746  * @dev: device
5747  * @upper_dev: upper device to check
5748  *
5749  * Find out if a device is linked to specified upper device and return true
5750  * in case it is. Note that this checks the entire upper device chain.
5751  * The caller must hold rcu lock.
5752  */
5753 
netdev_has_upper_dev_all_rcu(struct net_device * dev,struct net_device * upper_dev)5754 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5755 				  struct net_device *upper_dev)
5756 {
5757 	return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5758 					       upper_dev);
5759 }
5760 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5761 
5762 /**
5763  * netdev_has_any_upper_dev - Check if device is linked to some device
5764  * @dev: device
5765  *
5766  * Find out if a device is linked to an upper device and return true in case
5767  * it is. The caller must hold the RTNL lock.
5768  */
netdev_has_any_upper_dev(struct net_device * dev)5769 bool netdev_has_any_upper_dev(struct net_device *dev)
5770 {
5771 	ASSERT_RTNL();
5772 
5773 	return !list_empty(&dev->adj_list.upper);
5774 }
5775 EXPORT_SYMBOL(netdev_has_any_upper_dev);
5776 
5777 /**
5778  * netdev_master_upper_dev_get - Get master upper device
5779  * @dev: device
5780  *
5781  * Find a master upper device and return pointer to it or NULL in case
5782  * it's not there. The caller must hold the RTNL lock.
5783  */
netdev_master_upper_dev_get(struct net_device * dev)5784 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5785 {
5786 	struct netdev_adjacent *upper;
5787 
5788 	ASSERT_RTNL();
5789 
5790 	if (list_empty(&dev->adj_list.upper))
5791 		return NULL;
5792 
5793 	upper = list_first_entry(&dev->adj_list.upper,
5794 				 struct netdev_adjacent, list);
5795 	if (likely(upper->master))
5796 		return upper->dev;
5797 	return NULL;
5798 }
5799 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5800 
5801 /**
5802  * netdev_has_any_lower_dev - Check if device is linked to some device
5803  * @dev: device
5804  *
5805  * Find out if a device is linked to a lower device and return true in case
5806  * it is. The caller must hold the RTNL lock.
5807  */
netdev_has_any_lower_dev(struct net_device * dev)5808 static bool netdev_has_any_lower_dev(struct net_device *dev)
5809 {
5810 	ASSERT_RTNL();
5811 
5812 	return !list_empty(&dev->adj_list.lower);
5813 }
5814 
netdev_adjacent_get_private(struct list_head * adj_list)5815 void *netdev_adjacent_get_private(struct list_head *adj_list)
5816 {
5817 	struct netdev_adjacent *adj;
5818 
5819 	adj = list_entry(adj_list, struct netdev_adjacent, list);
5820 
5821 	return adj->private;
5822 }
5823 EXPORT_SYMBOL(netdev_adjacent_get_private);
5824 
5825 /**
5826  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5827  * @dev: device
5828  * @iter: list_head ** of the current position
5829  *
5830  * Gets the next device from the dev's upper list, starting from iter
5831  * position. The caller must hold RCU read lock.
5832  */
netdev_upper_get_next_dev_rcu(struct net_device * dev,struct list_head ** iter)5833 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5834 						 struct list_head **iter)
5835 {
5836 	struct netdev_adjacent *upper;
5837 
5838 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5839 
5840 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5841 
5842 	if (&upper->list == &dev->adj_list.upper)
5843 		return NULL;
5844 
5845 	*iter = &upper->list;
5846 
5847 	return upper->dev;
5848 }
5849 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5850 
netdev_next_upper_dev_rcu(struct net_device * dev,struct list_head ** iter)5851 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5852 						    struct list_head **iter)
5853 {
5854 	struct netdev_adjacent *upper;
5855 
5856 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5857 
5858 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5859 
5860 	if (&upper->list == &dev->adj_list.upper)
5861 		return NULL;
5862 
5863 	*iter = &upper->list;
5864 
5865 	return upper->dev;
5866 }
5867 
netdev_walk_all_upper_dev_rcu(struct net_device * dev,int (* fn)(struct net_device * dev,void * data),void * data)5868 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5869 				  int (*fn)(struct net_device *dev,
5870 					    void *data),
5871 				  void *data)
5872 {
5873 	struct net_device *udev;
5874 	struct list_head *iter;
5875 	int ret;
5876 
5877 	for (iter = &dev->adj_list.upper,
5878 	     udev = netdev_next_upper_dev_rcu(dev, &iter);
5879 	     udev;
5880 	     udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5881 		/* first is the upper device itself */
5882 		ret = fn(udev, data);
5883 		if (ret)
5884 			return ret;
5885 
5886 		/* then look at all of its upper devices */
5887 		ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5888 		if (ret)
5889 			return ret;
5890 	}
5891 
5892 	return 0;
5893 }
5894 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5895 
5896 /**
5897  * netdev_lower_get_next_private - Get the next ->private from the
5898  *				   lower neighbour list
5899  * @dev: device
5900  * @iter: list_head ** of the current position
5901  *
5902  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5903  * list, starting from iter position. The caller must hold either hold the
5904  * RTNL lock or its own locking that guarantees that the neighbour lower
5905  * list will remain unchanged.
5906  */
netdev_lower_get_next_private(struct net_device * dev,struct list_head ** iter)5907 void *netdev_lower_get_next_private(struct net_device *dev,
5908 				    struct list_head **iter)
5909 {
5910 	struct netdev_adjacent *lower;
5911 
5912 	lower = list_entry(*iter, struct netdev_adjacent, list);
5913 
5914 	if (&lower->list == &dev->adj_list.lower)
5915 		return NULL;
5916 
5917 	*iter = lower->list.next;
5918 
5919 	return lower->private;
5920 }
5921 EXPORT_SYMBOL(netdev_lower_get_next_private);
5922 
5923 /**
5924  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5925  *				       lower neighbour list, RCU
5926  *				       variant
5927  * @dev: device
5928  * @iter: list_head ** of the current position
5929  *
5930  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5931  * list, starting from iter position. The caller must hold RCU read lock.
5932  */
netdev_lower_get_next_private_rcu(struct net_device * dev,struct list_head ** iter)5933 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5934 					struct list_head **iter)
5935 {
5936 	struct netdev_adjacent *lower;
5937 
5938 	WARN_ON_ONCE(!rcu_read_lock_held());
5939 
5940 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5941 
5942 	if (&lower->list == &dev->adj_list.lower)
5943 		return NULL;
5944 
5945 	*iter = &lower->list;
5946 
5947 	return lower->private;
5948 }
5949 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5950 
5951 /**
5952  * netdev_lower_get_next - Get the next device from the lower neighbour
5953  *                         list
5954  * @dev: device
5955  * @iter: list_head ** of the current position
5956  *
5957  * Gets the next netdev_adjacent from the dev's lower neighbour
5958  * list, starting from iter position. The caller must hold RTNL lock or
5959  * its own locking that guarantees that the neighbour lower
5960  * list will remain unchanged.
5961  */
netdev_lower_get_next(struct net_device * dev,struct list_head ** iter)5962 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5963 {
5964 	struct netdev_adjacent *lower;
5965 
5966 	lower = list_entry(*iter, struct netdev_adjacent, list);
5967 
5968 	if (&lower->list == &dev->adj_list.lower)
5969 		return NULL;
5970 
5971 	*iter = lower->list.next;
5972 
5973 	return lower->dev;
5974 }
5975 EXPORT_SYMBOL(netdev_lower_get_next);
5976 
netdev_next_lower_dev(struct net_device * dev,struct list_head ** iter)5977 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5978 						struct list_head **iter)
5979 {
5980 	struct netdev_adjacent *lower;
5981 
5982 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5983 
5984 	if (&lower->list == &dev->adj_list.lower)
5985 		return NULL;
5986 
5987 	*iter = &lower->list;
5988 
5989 	return lower->dev;
5990 }
5991 
netdev_walk_all_lower_dev(struct net_device * dev,int (* fn)(struct net_device * dev,void * data),void * data)5992 int netdev_walk_all_lower_dev(struct net_device *dev,
5993 			      int (*fn)(struct net_device *dev,
5994 					void *data),
5995 			      void *data)
5996 {
5997 	struct net_device *ldev;
5998 	struct list_head *iter;
5999 	int ret;
6000 
6001 	for (iter = &dev->adj_list.lower,
6002 	     ldev = netdev_next_lower_dev(dev, &iter);
6003 	     ldev;
6004 	     ldev = netdev_next_lower_dev(dev, &iter)) {
6005 		/* first is the lower device itself */
6006 		ret = fn(ldev, data);
6007 		if (ret)
6008 			return ret;
6009 
6010 		/* then look at all of its lower devices */
6011 		ret = netdev_walk_all_lower_dev(ldev, fn, data);
6012 		if (ret)
6013 			return ret;
6014 	}
6015 
6016 	return 0;
6017 }
6018 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
6019 
netdev_next_lower_dev_rcu(struct net_device * dev,struct list_head ** iter)6020 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
6021 						    struct list_head **iter)
6022 {
6023 	struct netdev_adjacent *lower;
6024 
6025 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6026 	if (&lower->list == &dev->adj_list.lower)
6027 		return NULL;
6028 
6029 	*iter = &lower->list;
6030 
6031 	return lower->dev;
6032 }
6033 
netdev_walk_all_lower_dev_rcu(struct net_device * dev,int (* fn)(struct net_device * dev,void * data),void * data)6034 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
6035 				  int (*fn)(struct net_device *dev,
6036 					    void *data),
6037 				  void *data)
6038 {
6039 	struct net_device *ldev;
6040 	struct list_head *iter;
6041 	int ret;
6042 
6043 	for (iter = &dev->adj_list.lower,
6044 	     ldev = netdev_next_lower_dev_rcu(dev, &iter);
6045 	     ldev;
6046 	     ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
6047 		/* first is the lower device itself */
6048 		ret = fn(ldev, data);
6049 		if (ret)
6050 			return ret;
6051 
6052 		/* then look at all of its lower devices */
6053 		ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
6054 		if (ret)
6055 			return ret;
6056 	}
6057 
6058 	return 0;
6059 }
6060 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
6061 
6062 /**
6063  * netdev_lower_get_first_private_rcu - Get the first ->private from the
6064  *				       lower neighbour list, RCU
6065  *				       variant
6066  * @dev: device
6067  *
6068  * Gets the first netdev_adjacent->private from the dev's lower neighbour
6069  * list. The caller must hold RCU read lock.
6070  */
netdev_lower_get_first_private_rcu(struct net_device * dev)6071 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
6072 {
6073 	struct netdev_adjacent *lower;
6074 
6075 	lower = list_first_or_null_rcu(&dev->adj_list.lower,
6076 			struct netdev_adjacent, list);
6077 	if (lower)
6078 		return lower->private;
6079 	return NULL;
6080 }
6081 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
6082 
6083 /**
6084  * netdev_master_upper_dev_get_rcu - Get master upper device
6085  * @dev: device
6086  *
6087  * Find a master upper device and return pointer to it or NULL in case
6088  * it's not there. The caller must hold the RCU read lock.
6089  */
netdev_master_upper_dev_get_rcu(struct net_device * dev)6090 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
6091 {
6092 	struct netdev_adjacent *upper;
6093 
6094 	upper = list_first_or_null_rcu(&dev->adj_list.upper,
6095 				       struct netdev_adjacent, list);
6096 	if (upper && likely(upper->master))
6097 		return upper->dev;
6098 	return NULL;
6099 }
6100 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6101 
netdev_adjacent_sysfs_add(struct net_device * dev,struct net_device * adj_dev,struct list_head * dev_list)6102 static int netdev_adjacent_sysfs_add(struct net_device *dev,
6103 			      struct net_device *adj_dev,
6104 			      struct list_head *dev_list)
6105 {
6106 	char linkname[IFNAMSIZ+7];
6107 
6108 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
6109 		"upper_%s" : "lower_%s", adj_dev->name);
6110 	return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6111 				 linkname);
6112 }
netdev_adjacent_sysfs_del(struct net_device * dev,char * name,struct list_head * dev_list)6113 static void netdev_adjacent_sysfs_del(struct net_device *dev,
6114 			       char *name,
6115 			       struct list_head *dev_list)
6116 {
6117 	char linkname[IFNAMSIZ+7];
6118 
6119 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
6120 		"upper_%s" : "lower_%s", name);
6121 	sysfs_remove_link(&(dev->dev.kobj), linkname);
6122 }
6123 
netdev_adjacent_is_neigh_list(struct net_device * dev,struct net_device * adj_dev,struct list_head * dev_list)6124 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6125 						 struct net_device *adj_dev,
6126 						 struct list_head *dev_list)
6127 {
6128 	return (dev_list == &dev->adj_list.upper ||
6129 		dev_list == &dev->adj_list.lower) &&
6130 		net_eq(dev_net(dev), dev_net(adj_dev));
6131 }
6132 
__netdev_adjacent_dev_insert(struct net_device * dev,struct net_device * adj_dev,struct list_head * dev_list,void * private,bool master)6133 static int __netdev_adjacent_dev_insert(struct net_device *dev,
6134 					struct net_device *adj_dev,
6135 					struct list_head *dev_list,
6136 					void *private, bool master)
6137 {
6138 	struct netdev_adjacent *adj;
6139 	int ret;
6140 
6141 	adj = __netdev_find_adj(adj_dev, dev_list);
6142 
6143 	if (adj) {
6144 		adj->ref_nr += 1;
6145 		pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6146 			 dev->name, adj_dev->name, adj->ref_nr);
6147 
6148 		return 0;
6149 	}
6150 
6151 	adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6152 	if (!adj)
6153 		return -ENOMEM;
6154 
6155 	adj->dev = adj_dev;
6156 	adj->master = master;
6157 	adj->ref_nr = 1;
6158 	adj->private = private;
6159 	dev_hold(adj_dev);
6160 
6161 	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6162 		 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
6163 
6164 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
6165 		ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
6166 		if (ret)
6167 			goto free_adj;
6168 	}
6169 
6170 	/* Ensure that master link is always the first item in list. */
6171 	if (master) {
6172 		ret = sysfs_create_link(&(dev->dev.kobj),
6173 					&(adj_dev->dev.kobj), "master");
6174 		if (ret)
6175 			goto remove_symlinks;
6176 
6177 		list_add_rcu(&adj->list, dev_list);
6178 	} else {
6179 		list_add_tail_rcu(&adj->list, dev_list);
6180 	}
6181 
6182 	return 0;
6183 
6184 remove_symlinks:
6185 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6186 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6187 free_adj:
6188 	kfree(adj);
6189 	dev_put(adj_dev);
6190 
6191 	return ret;
6192 }
6193 
__netdev_adjacent_dev_remove(struct net_device * dev,struct net_device * adj_dev,u16 ref_nr,struct list_head * dev_list)6194 static void __netdev_adjacent_dev_remove(struct net_device *dev,
6195 					 struct net_device *adj_dev,
6196 					 u16 ref_nr,
6197 					 struct list_head *dev_list)
6198 {
6199 	struct netdev_adjacent *adj;
6200 
6201 	pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6202 		 dev->name, adj_dev->name, ref_nr);
6203 
6204 	adj = __netdev_find_adj(adj_dev, dev_list);
6205 
6206 	if (!adj) {
6207 		pr_err("Adjacency does not exist for device %s from %s\n",
6208 		       dev->name, adj_dev->name);
6209 		WARN_ON(1);
6210 		return;
6211 	}
6212 
6213 	if (adj->ref_nr > ref_nr) {
6214 		pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6215 			 dev->name, adj_dev->name, ref_nr,
6216 			 adj->ref_nr - ref_nr);
6217 		adj->ref_nr -= ref_nr;
6218 		return;
6219 	}
6220 
6221 	if (adj->master)
6222 		sysfs_remove_link(&(dev->dev.kobj), "master");
6223 
6224 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6225 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6226 
6227 	list_del_rcu(&adj->list);
6228 	pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
6229 		 adj_dev->name, dev->name, adj_dev->name);
6230 	dev_put(adj_dev);
6231 	kfree_rcu(adj, rcu);
6232 }
6233 
__netdev_adjacent_dev_link_lists(struct net_device * dev,struct net_device * upper_dev,struct list_head * up_list,struct list_head * down_list,void * private,bool master)6234 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6235 					    struct net_device *upper_dev,
6236 					    struct list_head *up_list,
6237 					    struct list_head *down_list,
6238 					    void *private, bool master)
6239 {
6240 	int ret;
6241 
6242 	ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
6243 					   private, master);
6244 	if (ret)
6245 		return ret;
6246 
6247 	ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
6248 					   private, false);
6249 	if (ret) {
6250 		__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
6251 		return ret;
6252 	}
6253 
6254 	return 0;
6255 }
6256 
__netdev_adjacent_dev_unlink_lists(struct net_device * dev,struct net_device * upper_dev,u16 ref_nr,struct list_head * up_list,struct list_head * down_list)6257 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
6258 					       struct net_device *upper_dev,
6259 					       u16 ref_nr,
6260 					       struct list_head *up_list,
6261 					       struct list_head *down_list)
6262 {
6263 	__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
6264 	__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
6265 }
6266 
__netdev_adjacent_dev_link_neighbour(struct net_device * dev,struct net_device * upper_dev,void * private,bool master)6267 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
6268 						struct net_device *upper_dev,
6269 						void *private, bool master)
6270 {
6271 	return __netdev_adjacent_dev_link_lists(dev, upper_dev,
6272 						&dev->adj_list.upper,
6273 						&upper_dev->adj_list.lower,
6274 						private, master);
6275 }
6276 
__netdev_adjacent_dev_unlink_neighbour(struct net_device * dev,struct net_device * upper_dev)6277 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
6278 						   struct net_device *upper_dev)
6279 {
6280 	__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
6281 					   &dev->adj_list.upper,
6282 					   &upper_dev->adj_list.lower);
6283 }
6284 
__netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev,bool master,void * upper_priv,void * upper_info)6285 static int __netdev_upper_dev_link(struct net_device *dev,
6286 				   struct net_device *upper_dev, bool master,
6287 				   void *upper_priv, void *upper_info)
6288 {
6289 	struct netdev_notifier_changeupper_info changeupper_info;
6290 	int ret = 0;
6291 
6292 	ASSERT_RTNL();
6293 
6294 	if (dev == upper_dev)
6295 		return -EBUSY;
6296 
6297 	/* To prevent loops, check if dev is not upper device to upper_dev. */
6298 	if (netdev_has_upper_dev(upper_dev, dev))
6299 		return -EBUSY;
6300 
6301 	if (netdev_has_upper_dev(dev, upper_dev))
6302 		return -EEXIST;
6303 
6304 	if (master && netdev_master_upper_dev_get(dev))
6305 		return -EBUSY;
6306 
6307 	changeupper_info.upper_dev = upper_dev;
6308 	changeupper_info.master = master;
6309 	changeupper_info.linking = true;
6310 	changeupper_info.upper_info = upper_info;
6311 
6312 	ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6313 					    &changeupper_info.info);
6314 	ret = notifier_to_errno(ret);
6315 	if (ret)
6316 		return ret;
6317 
6318 	ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
6319 						   master);
6320 	if (ret)
6321 		return ret;
6322 
6323 	ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6324 					    &changeupper_info.info);
6325 	ret = notifier_to_errno(ret);
6326 	if (ret)
6327 		goto rollback;
6328 
6329 	return 0;
6330 
6331 rollback:
6332 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6333 
6334 	return ret;
6335 }
6336 
6337 /**
6338  * netdev_upper_dev_link - Add a link to the upper device
6339  * @dev: device
6340  * @upper_dev: new upper device
6341  *
6342  * Adds a link to device which is upper to this one. The caller must hold
6343  * the RTNL lock. On a failure a negative errno code is returned.
6344  * On success the reference counts are adjusted and the function
6345  * returns zero.
6346  */
netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev)6347 int netdev_upper_dev_link(struct net_device *dev,
6348 			  struct net_device *upper_dev)
6349 {
6350 	return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
6351 }
6352 EXPORT_SYMBOL(netdev_upper_dev_link);
6353 
6354 /**
6355  * netdev_master_upper_dev_link - Add a master link to the upper device
6356  * @dev: device
6357  * @upper_dev: new upper device
6358  * @upper_priv: upper device private
6359  * @upper_info: upper info to be passed down via notifier
6360  *
6361  * Adds a link to device which is upper to this one. In this case, only
6362  * one master upper device can be linked, although other non-master devices
6363  * might be linked as well. The caller must hold the RTNL lock.
6364  * On a failure a negative errno code is returned. On success the reference
6365  * counts are adjusted and the function returns zero.
6366  */
netdev_master_upper_dev_link(struct net_device * dev,struct net_device * upper_dev,void * upper_priv,void * upper_info)6367 int netdev_master_upper_dev_link(struct net_device *dev,
6368 				 struct net_device *upper_dev,
6369 				 void *upper_priv, void *upper_info)
6370 {
6371 	return __netdev_upper_dev_link(dev, upper_dev, true,
6372 				       upper_priv, upper_info);
6373 }
6374 EXPORT_SYMBOL(netdev_master_upper_dev_link);
6375 
6376 /**
6377  * netdev_upper_dev_unlink - Removes a link to upper device
6378  * @dev: device
6379  * @upper_dev: new upper device
6380  *
6381  * Removes a link to device which is upper to this one. The caller must hold
6382  * the RTNL lock.
6383  */
netdev_upper_dev_unlink(struct net_device * dev,struct net_device * upper_dev)6384 void netdev_upper_dev_unlink(struct net_device *dev,
6385 			     struct net_device *upper_dev)
6386 {
6387 	struct netdev_notifier_changeupper_info changeupper_info;
6388 
6389 	ASSERT_RTNL();
6390 
6391 	changeupper_info.upper_dev = upper_dev;
6392 	changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
6393 	changeupper_info.linking = false;
6394 
6395 	call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6396 				      &changeupper_info.info);
6397 
6398 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6399 
6400 	call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6401 				      &changeupper_info.info);
6402 }
6403 EXPORT_SYMBOL(netdev_upper_dev_unlink);
6404 
6405 /**
6406  * netdev_bonding_info_change - Dispatch event about slave change
6407  * @dev: device
6408  * @bonding_info: info to dispatch
6409  *
6410  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
6411  * The caller must hold the RTNL lock.
6412  */
netdev_bonding_info_change(struct net_device * dev,struct netdev_bonding_info * bonding_info)6413 void netdev_bonding_info_change(struct net_device *dev,
6414 				struct netdev_bonding_info *bonding_info)
6415 {
6416 	struct netdev_notifier_bonding_info	info;
6417 
6418 	memcpy(&info.bonding_info, bonding_info,
6419 	       sizeof(struct netdev_bonding_info));
6420 	call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
6421 				      &info.info);
6422 }
6423 EXPORT_SYMBOL(netdev_bonding_info_change);
6424 
netdev_adjacent_add_links(struct net_device * dev)6425 static void netdev_adjacent_add_links(struct net_device *dev)
6426 {
6427 	struct netdev_adjacent *iter;
6428 
6429 	struct net *net = dev_net(dev);
6430 
6431 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6432 		if (!net_eq(net, dev_net(iter->dev)))
6433 			continue;
6434 		netdev_adjacent_sysfs_add(iter->dev, dev,
6435 					  &iter->dev->adj_list.lower);
6436 		netdev_adjacent_sysfs_add(dev, iter->dev,
6437 					  &dev->adj_list.upper);
6438 	}
6439 
6440 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6441 		if (!net_eq(net, dev_net(iter->dev)))
6442 			continue;
6443 		netdev_adjacent_sysfs_add(iter->dev, dev,
6444 					  &iter->dev->adj_list.upper);
6445 		netdev_adjacent_sysfs_add(dev, iter->dev,
6446 					  &dev->adj_list.lower);
6447 	}
6448 }
6449 
netdev_adjacent_del_links(struct net_device * dev)6450 static void netdev_adjacent_del_links(struct net_device *dev)
6451 {
6452 	struct netdev_adjacent *iter;
6453 
6454 	struct net *net = dev_net(dev);
6455 
6456 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6457 		if (!net_eq(net, dev_net(iter->dev)))
6458 			continue;
6459 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
6460 					  &iter->dev->adj_list.lower);
6461 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
6462 					  &dev->adj_list.upper);
6463 	}
6464 
6465 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6466 		if (!net_eq(net, dev_net(iter->dev)))
6467 			continue;
6468 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
6469 					  &iter->dev->adj_list.upper);
6470 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
6471 					  &dev->adj_list.lower);
6472 	}
6473 }
6474 
netdev_adjacent_rename_links(struct net_device * dev,char * oldname)6475 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
6476 {
6477 	struct netdev_adjacent *iter;
6478 
6479 	struct net *net = dev_net(dev);
6480 
6481 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6482 		if (!net_eq(net, dev_net(iter->dev)))
6483 			continue;
6484 		netdev_adjacent_sysfs_del(iter->dev, oldname,
6485 					  &iter->dev->adj_list.lower);
6486 		netdev_adjacent_sysfs_add(iter->dev, dev,
6487 					  &iter->dev->adj_list.lower);
6488 	}
6489 
6490 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6491 		if (!net_eq(net, dev_net(iter->dev)))
6492 			continue;
6493 		netdev_adjacent_sysfs_del(iter->dev, oldname,
6494 					  &iter->dev->adj_list.upper);
6495 		netdev_adjacent_sysfs_add(iter->dev, dev,
6496 					  &iter->dev->adj_list.upper);
6497 	}
6498 }
6499 
netdev_lower_dev_get_private(struct net_device * dev,struct net_device * lower_dev)6500 void *netdev_lower_dev_get_private(struct net_device *dev,
6501 				   struct net_device *lower_dev)
6502 {
6503 	struct netdev_adjacent *lower;
6504 
6505 	if (!lower_dev)
6506 		return NULL;
6507 	lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
6508 	if (!lower)
6509 		return NULL;
6510 
6511 	return lower->private;
6512 }
6513 EXPORT_SYMBOL(netdev_lower_dev_get_private);
6514 
6515 
dev_get_nest_level(struct net_device * dev)6516 int dev_get_nest_level(struct net_device *dev)
6517 {
6518 	struct net_device *lower = NULL;
6519 	struct list_head *iter;
6520 	int max_nest = -1;
6521 	int nest;
6522 
6523 	ASSERT_RTNL();
6524 
6525 	netdev_for_each_lower_dev(dev, lower, iter) {
6526 		nest = dev_get_nest_level(lower);
6527 		if (max_nest < nest)
6528 			max_nest = nest;
6529 	}
6530 
6531 	return max_nest + 1;
6532 }
6533 EXPORT_SYMBOL(dev_get_nest_level);
6534 
6535 /**
6536  * netdev_lower_change - Dispatch event about lower device state change
6537  * @lower_dev: device
6538  * @lower_state_info: state to dispatch
6539  *
6540  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6541  * The caller must hold the RTNL lock.
6542  */
netdev_lower_state_changed(struct net_device * lower_dev,void * lower_state_info)6543 void netdev_lower_state_changed(struct net_device *lower_dev,
6544 				void *lower_state_info)
6545 {
6546 	struct netdev_notifier_changelowerstate_info changelowerstate_info;
6547 
6548 	ASSERT_RTNL();
6549 	changelowerstate_info.lower_state_info = lower_state_info;
6550 	call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6551 				      &changelowerstate_info.info);
6552 }
6553 EXPORT_SYMBOL(netdev_lower_state_changed);
6554 
dev_change_rx_flags(struct net_device * dev,int flags)6555 static void dev_change_rx_flags(struct net_device *dev, int flags)
6556 {
6557 	const struct net_device_ops *ops = dev->netdev_ops;
6558 
6559 	if (ops->ndo_change_rx_flags)
6560 		ops->ndo_change_rx_flags(dev, flags);
6561 }
6562 
__dev_set_promiscuity(struct net_device * dev,int inc,bool notify)6563 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6564 {
6565 	unsigned int old_flags = dev->flags;
6566 	kuid_t uid;
6567 	kgid_t gid;
6568 
6569 	ASSERT_RTNL();
6570 
6571 	dev->flags |= IFF_PROMISC;
6572 	dev->promiscuity += inc;
6573 	if (dev->promiscuity == 0) {
6574 		/*
6575 		 * Avoid overflow.
6576 		 * If inc causes overflow, untouch promisc and return error.
6577 		 */
6578 		if (inc < 0)
6579 			dev->flags &= ~IFF_PROMISC;
6580 		else {
6581 			dev->promiscuity -= inc;
6582 			pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6583 				dev->name);
6584 			return -EOVERFLOW;
6585 		}
6586 	}
6587 	if (dev->flags != old_flags) {
6588 		pr_info("device %s %s promiscuous mode\n",
6589 			dev->name,
6590 			dev->flags & IFF_PROMISC ? "entered" : "left");
6591 		if (audit_enabled) {
6592 			current_uid_gid(&uid, &gid);
6593 			audit_log(current->audit_context, GFP_ATOMIC,
6594 				AUDIT_ANOM_PROMISCUOUS,
6595 				"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6596 				dev->name, (dev->flags & IFF_PROMISC),
6597 				(old_flags & IFF_PROMISC),
6598 				from_kuid(&init_user_ns, audit_get_loginuid(current)),
6599 				from_kuid(&init_user_ns, uid),
6600 				from_kgid(&init_user_ns, gid),
6601 				audit_get_sessionid(current));
6602 		}
6603 
6604 		dev_change_rx_flags(dev, IFF_PROMISC);
6605 	}
6606 	if (notify)
6607 		__dev_notify_flags(dev, old_flags, IFF_PROMISC);
6608 	return 0;
6609 }
6610 
6611 /**
6612  *	dev_set_promiscuity	- update promiscuity count on a device
6613  *	@dev: device
6614  *	@inc: modifier
6615  *
6616  *	Add or remove promiscuity from a device. While the count in the device
6617  *	remains above zero the interface remains promiscuous. Once it hits zero
6618  *	the device reverts back to normal filtering operation. A negative inc
6619  *	value is used to drop promiscuity on the device.
6620  *	Return 0 if successful or a negative errno code on error.
6621  */
dev_set_promiscuity(struct net_device * dev,int inc)6622 int dev_set_promiscuity(struct net_device *dev, int inc)
6623 {
6624 	unsigned int old_flags = dev->flags;
6625 	int err;
6626 
6627 	err = __dev_set_promiscuity(dev, inc, true);
6628 	if (err < 0)
6629 		return err;
6630 	if (dev->flags != old_flags)
6631 		dev_set_rx_mode(dev);
6632 	return err;
6633 }
6634 EXPORT_SYMBOL(dev_set_promiscuity);
6635 
__dev_set_allmulti(struct net_device * dev,int inc,bool notify)6636 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6637 {
6638 	unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6639 
6640 	ASSERT_RTNL();
6641 
6642 	dev->flags |= IFF_ALLMULTI;
6643 	dev->allmulti += inc;
6644 	if (dev->allmulti == 0) {
6645 		/*
6646 		 * Avoid overflow.
6647 		 * If inc causes overflow, untouch allmulti and return error.
6648 		 */
6649 		if (inc < 0)
6650 			dev->flags &= ~IFF_ALLMULTI;
6651 		else {
6652 			dev->allmulti -= inc;
6653 			pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6654 				dev->name);
6655 			return -EOVERFLOW;
6656 		}
6657 	}
6658 	if (dev->flags ^ old_flags) {
6659 		dev_change_rx_flags(dev, IFF_ALLMULTI);
6660 		dev_set_rx_mode(dev);
6661 		if (notify)
6662 			__dev_notify_flags(dev, old_flags,
6663 					   dev->gflags ^ old_gflags);
6664 	}
6665 	return 0;
6666 }
6667 
6668 /**
6669  *	dev_set_allmulti	- update allmulti count on a device
6670  *	@dev: device
6671  *	@inc: modifier
6672  *
6673  *	Add or remove reception of all multicast frames to a device. While the
6674  *	count in the device remains above zero the interface remains listening
6675  *	to all interfaces. Once it hits zero the device reverts back to normal
6676  *	filtering operation. A negative @inc value is used to drop the counter
6677  *	when releasing a resource needing all multicasts.
6678  *	Return 0 if successful or a negative errno code on error.
6679  */
6680 
dev_set_allmulti(struct net_device * dev,int inc)6681 int dev_set_allmulti(struct net_device *dev, int inc)
6682 {
6683 	return __dev_set_allmulti(dev, inc, true);
6684 }
6685 EXPORT_SYMBOL(dev_set_allmulti);
6686 
6687 /*
6688  *	Upload unicast and multicast address lists to device and
6689  *	configure RX filtering. When the device doesn't support unicast
6690  *	filtering it is put in promiscuous mode while unicast addresses
6691  *	are present.
6692  */
__dev_set_rx_mode(struct net_device * dev)6693 void __dev_set_rx_mode(struct net_device *dev)
6694 {
6695 	const struct net_device_ops *ops = dev->netdev_ops;
6696 
6697 	/* dev_open will call this function so the list will stay sane. */
6698 	if (!(dev->flags&IFF_UP))
6699 		return;
6700 
6701 	if (!netif_device_present(dev))
6702 		return;
6703 
6704 	if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6705 		/* Unicast addresses changes may only happen under the rtnl,
6706 		 * therefore calling __dev_set_promiscuity here is safe.
6707 		 */
6708 		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6709 			__dev_set_promiscuity(dev, 1, false);
6710 			dev->uc_promisc = true;
6711 		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6712 			__dev_set_promiscuity(dev, -1, false);
6713 			dev->uc_promisc = false;
6714 		}
6715 	}
6716 
6717 	if (ops->ndo_set_rx_mode)
6718 		ops->ndo_set_rx_mode(dev);
6719 }
6720 
dev_set_rx_mode(struct net_device * dev)6721 void dev_set_rx_mode(struct net_device *dev)
6722 {
6723 	netif_addr_lock_bh(dev);
6724 	__dev_set_rx_mode(dev);
6725 	netif_addr_unlock_bh(dev);
6726 }
6727 
6728 /**
6729  *	dev_get_flags - get flags reported to userspace
6730  *	@dev: device
6731  *
6732  *	Get the combination of flag bits exported through APIs to userspace.
6733  */
dev_get_flags(const struct net_device * dev)6734 unsigned int dev_get_flags(const struct net_device *dev)
6735 {
6736 	unsigned int flags;
6737 
6738 	flags = (dev->flags & ~(IFF_PROMISC |
6739 				IFF_ALLMULTI |
6740 				IFF_RUNNING |
6741 				IFF_LOWER_UP |
6742 				IFF_DORMANT)) |
6743 		(dev->gflags & (IFF_PROMISC |
6744 				IFF_ALLMULTI));
6745 
6746 	if (netif_running(dev)) {
6747 		if (netif_oper_up(dev))
6748 			flags |= IFF_RUNNING;
6749 		if (netif_carrier_ok(dev))
6750 			flags |= IFF_LOWER_UP;
6751 		if (netif_dormant(dev))
6752 			flags |= IFF_DORMANT;
6753 	}
6754 
6755 	return flags;
6756 }
6757 EXPORT_SYMBOL(dev_get_flags);
6758 
__dev_change_flags(struct net_device * dev,unsigned int flags)6759 int __dev_change_flags(struct net_device *dev, unsigned int flags)
6760 {
6761 	unsigned int old_flags = dev->flags;
6762 	int ret;
6763 
6764 	ASSERT_RTNL();
6765 
6766 	/*
6767 	 *	Set the flags on our device.
6768 	 */
6769 
6770 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6771 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6772 			       IFF_AUTOMEDIA)) |
6773 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6774 				    IFF_ALLMULTI));
6775 
6776 	/*
6777 	 *	Load in the correct multicast list now the flags have changed.
6778 	 */
6779 
6780 	if ((old_flags ^ flags) & IFF_MULTICAST)
6781 		dev_change_rx_flags(dev, IFF_MULTICAST);
6782 
6783 	dev_set_rx_mode(dev);
6784 
6785 	/*
6786 	 *	Have we downed the interface. We handle IFF_UP ourselves
6787 	 *	according to user attempts to set it, rather than blindly
6788 	 *	setting it.
6789 	 */
6790 
6791 	ret = 0;
6792 	if ((old_flags ^ flags) & IFF_UP) {
6793 		if (old_flags & IFF_UP)
6794 			__dev_close(dev);
6795 		else
6796 			ret = __dev_open(dev);
6797 	}
6798 
6799 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
6800 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
6801 		unsigned int old_flags = dev->flags;
6802 
6803 		dev->gflags ^= IFF_PROMISC;
6804 
6805 		if (__dev_set_promiscuity(dev, inc, false) >= 0)
6806 			if (dev->flags != old_flags)
6807 				dev_set_rx_mode(dev);
6808 	}
6809 
6810 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6811 	 * is important. Some (broken) drivers set IFF_PROMISC, when
6812 	 * IFF_ALLMULTI is requested not asking us and not reporting.
6813 	 */
6814 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6815 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6816 
6817 		dev->gflags ^= IFF_ALLMULTI;
6818 		__dev_set_allmulti(dev, inc, false);
6819 	}
6820 
6821 	return ret;
6822 }
6823 
__dev_notify_flags(struct net_device * dev,unsigned int old_flags,unsigned int gchanges)6824 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6825 			unsigned int gchanges)
6826 {
6827 	unsigned int changes = dev->flags ^ old_flags;
6828 
6829 	if (gchanges)
6830 		rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6831 
6832 	if (changes & IFF_UP) {
6833 		if (dev->flags & IFF_UP)
6834 			call_netdevice_notifiers(NETDEV_UP, dev);
6835 		else
6836 			call_netdevice_notifiers(NETDEV_DOWN, dev);
6837 	}
6838 
6839 	if (dev->flags & IFF_UP &&
6840 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6841 		struct netdev_notifier_change_info change_info;
6842 
6843 		change_info.flags_changed = changes;
6844 		call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6845 					      &change_info.info);
6846 	}
6847 }
6848 
6849 /**
6850  *	dev_change_flags - change device settings
6851  *	@dev: device
6852  *	@flags: device state flags
6853  *
6854  *	Change settings on device based state flags. The flags are
6855  *	in the userspace exported format.
6856  */
dev_change_flags(struct net_device * dev,unsigned int flags)6857 int dev_change_flags(struct net_device *dev, unsigned int flags)
6858 {
6859 	int ret;
6860 	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6861 
6862 	ret = __dev_change_flags(dev, flags);
6863 	if (ret < 0)
6864 		return ret;
6865 
6866 	changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6867 	__dev_notify_flags(dev, old_flags, changes);
6868 	return ret;
6869 }
6870 EXPORT_SYMBOL(dev_change_flags);
6871 
__dev_set_mtu(struct net_device * dev,int new_mtu)6872 int __dev_set_mtu(struct net_device *dev, int new_mtu)
6873 {
6874 	const struct net_device_ops *ops = dev->netdev_ops;
6875 
6876 	if (ops->ndo_change_mtu)
6877 		return ops->ndo_change_mtu(dev, new_mtu);
6878 
6879 	/* Pairs with all the lockless reads of dev->mtu in the stack */
6880 	WRITE_ONCE(dev->mtu, new_mtu);
6881 	return 0;
6882 }
6883 EXPORT_SYMBOL(__dev_set_mtu);
6884 
6885 /**
6886  *	dev_set_mtu - Change maximum transfer unit
6887  *	@dev: device
6888  *	@new_mtu: new transfer unit
6889  *
6890  *	Change the maximum transfer size of the network device.
6891  */
dev_set_mtu(struct net_device * dev,int new_mtu)6892 int dev_set_mtu(struct net_device *dev, int new_mtu)
6893 {
6894 	int err, orig_mtu;
6895 
6896 	if (new_mtu == dev->mtu)
6897 		return 0;
6898 
6899 	err = dev_validate_mtu(dev, new_mtu);
6900 	if (err)
6901 		return err;
6902 
6903 	if (!netif_device_present(dev))
6904 		return -ENODEV;
6905 
6906 	err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6907 	err = notifier_to_errno(err);
6908 	if (err)
6909 		return err;
6910 
6911 	orig_mtu = dev->mtu;
6912 	err = __dev_set_mtu(dev, new_mtu);
6913 
6914 	if (!err) {
6915 		err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
6916 						   orig_mtu);
6917 		err = notifier_to_errno(err);
6918 		if (err) {
6919 			/* setting mtu back and notifying everyone again,
6920 			 * so that they have a chance to revert changes.
6921 			 */
6922 			__dev_set_mtu(dev, orig_mtu);
6923 			call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
6924 						     new_mtu);
6925 		}
6926 	}
6927 	return err;
6928 }
6929 EXPORT_SYMBOL(dev_set_mtu);
6930 
6931 /**
6932  *	dev_set_group - Change group this device belongs to
6933  *	@dev: device
6934  *	@new_group: group this device should belong to
6935  */
dev_set_group(struct net_device * dev,int new_group)6936 void dev_set_group(struct net_device *dev, int new_group)
6937 {
6938 	dev->group = new_group;
6939 }
6940 EXPORT_SYMBOL(dev_set_group);
6941 
6942 /**
6943  *	dev_set_mac_address - Change Media Access Control Address
6944  *	@dev: device
6945  *	@sa: new address
6946  *
6947  *	Change the hardware (MAC) address of the device
6948  */
dev_set_mac_address(struct net_device * dev,struct sockaddr * sa)6949 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6950 {
6951 	const struct net_device_ops *ops = dev->netdev_ops;
6952 	int err;
6953 
6954 	if (!ops->ndo_set_mac_address)
6955 		return -EOPNOTSUPP;
6956 	if (sa->sa_family != dev->type)
6957 		return -EINVAL;
6958 	if (!netif_device_present(dev))
6959 		return -ENODEV;
6960 	err = ops->ndo_set_mac_address(dev, sa);
6961 	if (err)
6962 		return err;
6963 	dev->addr_assign_type = NET_ADDR_SET;
6964 	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6965 	add_device_randomness(dev->dev_addr, dev->addr_len);
6966 	return 0;
6967 }
6968 EXPORT_SYMBOL(dev_set_mac_address);
6969 
6970 /**
6971  *	dev_change_carrier - Change device carrier
6972  *	@dev: device
6973  *	@new_carrier: new value
6974  *
6975  *	Change device carrier
6976  */
dev_change_carrier(struct net_device * dev,bool new_carrier)6977 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6978 {
6979 	const struct net_device_ops *ops = dev->netdev_ops;
6980 
6981 	if (!ops->ndo_change_carrier)
6982 		return -EOPNOTSUPP;
6983 	if (!netif_device_present(dev))
6984 		return -ENODEV;
6985 	return ops->ndo_change_carrier(dev, new_carrier);
6986 }
6987 EXPORT_SYMBOL(dev_change_carrier);
6988 
6989 /**
6990  *	dev_get_phys_port_id - Get device physical port ID
6991  *	@dev: device
6992  *	@ppid: port ID
6993  *
6994  *	Get device physical port ID
6995  */
dev_get_phys_port_id(struct net_device * dev,struct netdev_phys_item_id * ppid)6996 int dev_get_phys_port_id(struct net_device *dev,
6997 			 struct netdev_phys_item_id *ppid)
6998 {
6999 	const struct net_device_ops *ops = dev->netdev_ops;
7000 
7001 	if (!ops->ndo_get_phys_port_id)
7002 		return -EOPNOTSUPP;
7003 	return ops->ndo_get_phys_port_id(dev, ppid);
7004 }
7005 EXPORT_SYMBOL(dev_get_phys_port_id);
7006 
7007 /**
7008  *	dev_get_phys_port_name - Get device physical port name
7009  *	@dev: device
7010  *	@name: port name
7011  *	@len: limit of bytes to copy to name
7012  *
7013  *	Get device physical port name
7014  */
dev_get_phys_port_name(struct net_device * dev,char * name,size_t len)7015 int dev_get_phys_port_name(struct net_device *dev,
7016 			   char *name, size_t len)
7017 {
7018 	const struct net_device_ops *ops = dev->netdev_ops;
7019 
7020 	if (!ops->ndo_get_phys_port_name)
7021 		return -EOPNOTSUPP;
7022 	return ops->ndo_get_phys_port_name(dev, name, len);
7023 }
7024 EXPORT_SYMBOL(dev_get_phys_port_name);
7025 
7026 /**
7027  *	dev_change_proto_down - update protocol port state information
7028  *	@dev: device
7029  *	@proto_down: new value
7030  *
7031  *	This info can be used by switch drivers to set the phys state of the
7032  *	port.
7033  */
dev_change_proto_down(struct net_device * dev,bool proto_down)7034 int dev_change_proto_down(struct net_device *dev, bool proto_down)
7035 {
7036 	const struct net_device_ops *ops = dev->netdev_ops;
7037 
7038 	if (!ops->ndo_change_proto_down)
7039 		return -EOPNOTSUPP;
7040 	if (!netif_device_present(dev))
7041 		return -ENODEV;
7042 	return ops->ndo_change_proto_down(dev, proto_down);
7043 }
7044 EXPORT_SYMBOL(dev_change_proto_down);
7045 
__dev_xdp_attached(struct net_device * dev,xdp_op_t xdp_op,u32 * prog_id)7046 u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id)
7047 {
7048 	struct netdev_xdp xdp;
7049 
7050 	memset(&xdp, 0, sizeof(xdp));
7051 	xdp.command = XDP_QUERY_PROG;
7052 
7053 	/* Query must always succeed. */
7054 	WARN_ON(xdp_op(dev, &xdp) < 0);
7055 	if (prog_id)
7056 		*prog_id = xdp.prog_id;
7057 
7058 	return xdp.prog_attached;
7059 }
7060 
dev_xdp_install(struct net_device * dev,xdp_op_t xdp_op,struct netlink_ext_ack * extack,u32 flags,struct bpf_prog * prog)7061 static int dev_xdp_install(struct net_device *dev, xdp_op_t xdp_op,
7062 			   struct netlink_ext_ack *extack, u32 flags,
7063 			   struct bpf_prog *prog)
7064 {
7065 	struct netdev_xdp xdp;
7066 
7067 	memset(&xdp, 0, sizeof(xdp));
7068 	if (flags & XDP_FLAGS_HW_MODE)
7069 		xdp.command = XDP_SETUP_PROG_HW;
7070 	else
7071 		xdp.command = XDP_SETUP_PROG;
7072 	xdp.extack = extack;
7073 	xdp.flags = flags;
7074 	xdp.prog = prog;
7075 
7076 	return xdp_op(dev, &xdp);
7077 }
7078 
7079 /**
7080  *	dev_change_xdp_fd - set or clear a bpf program for a device rx path
7081  *	@dev: device
7082  *	@extack: netlink extended ack
7083  *	@fd: new program fd or negative value to clear
7084  *	@flags: xdp-related flags
7085  *
7086  *	Set or clear a bpf program for a device
7087  */
dev_change_xdp_fd(struct net_device * dev,struct netlink_ext_ack * extack,int fd,u32 flags)7088 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
7089 		      int fd, u32 flags)
7090 {
7091 	const struct net_device_ops *ops = dev->netdev_ops;
7092 	struct bpf_prog *prog = NULL;
7093 	xdp_op_t xdp_op, xdp_chk;
7094 	int err;
7095 
7096 	ASSERT_RTNL();
7097 
7098 	xdp_op = xdp_chk = ops->ndo_xdp;
7099 	if (!xdp_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
7100 		return -EOPNOTSUPP;
7101 	if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
7102 		xdp_op = generic_xdp_install;
7103 	if (xdp_op == xdp_chk)
7104 		xdp_chk = generic_xdp_install;
7105 
7106 	if (fd >= 0) {
7107 		if (xdp_chk && __dev_xdp_attached(dev, xdp_chk, NULL))
7108 			return -EEXIST;
7109 		if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
7110 		    __dev_xdp_attached(dev, xdp_op, NULL))
7111 			return -EBUSY;
7112 
7113 		prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
7114 		if (IS_ERR(prog))
7115 			return PTR_ERR(prog);
7116 	}
7117 
7118 	err = dev_xdp_install(dev, xdp_op, extack, flags, prog);
7119 	if (err < 0 && prog)
7120 		bpf_prog_put(prog);
7121 
7122 	return err;
7123 }
7124 
7125 /**
7126  *	dev_new_index	-	allocate an ifindex
7127  *	@net: the applicable net namespace
7128  *
7129  *	Returns a suitable unique value for a new device interface
7130  *	number.  The caller must hold the rtnl semaphore or the
7131  *	dev_base_lock to be sure it remains unique.
7132  */
dev_new_index(struct net * net)7133 static int dev_new_index(struct net *net)
7134 {
7135 	int ifindex = net->ifindex;
7136 
7137 	for (;;) {
7138 		if (++ifindex <= 0)
7139 			ifindex = 1;
7140 		if (!__dev_get_by_index(net, ifindex))
7141 			return net->ifindex = ifindex;
7142 	}
7143 }
7144 
7145 /* Delayed registration/unregisteration */
7146 static LIST_HEAD(net_todo_list);
7147 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
7148 
net_set_todo(struct net_device * dev)7149 static void net_set_todo(struct net_device *dev)
7150 {
7151 	list_add_tail(&dev->todo_list, &net_todo_list);
7152 	dev_net(dev)->dev_unreg_count++;
7153 }
7154 
rollback_registered_many(struct list_head * head)7155 static void rollback_registered_many(struct list_head *head)
7156 {
7157 	struct net_device *dev, *tmp;
7158 	LIST_HEAD(close_head);
7159 
7160 	BUG_ON(dev_boot_phase);
7161 	ASSERT_RTNL();
7162 
7163 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
7164 		/* Some devices call without registering
7165 		 * for initialization unwind. Remove those
7166 		 * devices and proceed with the remaining.
7167 		 */
7168 		if (dev->reg_state == NETREG_UNINITIALIZED) {
7169 			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
7170 				 dev->name, dev);
7171 
7172 			WARN_ON(1);
7173 			list_del(&dev->unreg_list);
7174 			continue;
7175 		}
7176 		dev->dismantle = true;
7177 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
7178 	}
7179 
7180 	/* If device is running, close it first. */
7181 	list_for_each_entry(dev, head, unreg_list)
7182 		list_add_tail(&dev->close_list, &close_head);
7183 	dev_close_many(&close_head, true);
7184 
7185 	list_for_each_entry(dev, head, unreg_list) {
7186 		/* And unlink it from device chain. */
7187 		unlist_netdevice(dev);
7188 
7189 		dev->reg_state = NETREG_UNREGISTERING;
7190 	}
7191 	flush_all_backlogs();
7192 
7193 	synchronize_net();
7194 
7195 	list_for_each_entry(dev, head, unreg_list) {
7196 		struct sk_buff *skb = NULL;
7197 
7198 		/* Shutdown queueing discipline. */
7199 		dev_shutdown(dev);
7200 
7201 
7202 		/* Notify protocols, that we are about to destroy
7203 		 * this device. They should clean all the things.
7204 		 */
7205 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7206 
7207 		if (!dev->rtnl_link_ops ||
7208 		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7209 			skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
7210 						     GFP_KERNEL);
7211 
7212 		/*
7213 		 *	Flush the unicast and multicast chains
7214 		 */
7215 		dev_uc_flush(dev);
7216 		dev_mc_flush(dev);
7217 
7218 		if (dev->netdev_ops->ndo_uninit)
7219 			dev->netdev_ops->ndo_uninit(dev);
7220 
7221 		if (skb)
7222 			rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
7223 
7224 		/* Notifier chain MUST detach us all upper devices. */
7225 		WARN_ON(netdev_has_any_upper_dev(dev));
7226 		WARN_ON(netdev_has_any_lower_dev(dev));
7227 
7228 		/* Remove entries from kobject tree */
7229 		netdev_unregister_kobject(dev);
7230 #ifdef CONFIG_XPS
7231 		/* Remove XPS queueing entries */
7232 		netif_reset_xps_queues_gt(dev, 0);
7233 #endif
7234 	}
7235 
7236 	synchronize_net();
7237 
7238 	list_for_each_entry(dev, head, unreg_list)
7239 		dev_put(dev);
7240 }
7241 
rollback_registered(struct net_device * dev)7242 static void rollback_registered(struct net_device *dev)
7243 {
7244 	LIST_HEAD(single);
7245 
7246 	list_add(&dev->unreg_list, &single);
7247 	rollback_registered_many(&single);
7248 	list_del(&single);
7249 }
7250 
netdev_sync_upper_features(struct net_device * lower,struct net_device * upper,netdev_features_t features)7251 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
7252 	struct net_device *upper, netdev_features_t features)
7253 {
7254 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7255 	netdev_features_t feature;
7256 	int feature_bit;
7257 
7258 	for_each_netdev_feature(upper_disables, feature_bit) {
7259 		feature = __NETIF_F_BIT(feature_bit);
7260 		if (!(upper->wanted_features & feature)
7261 		    && (features & feature)) {
7262 			netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
7263 				   &feature, upper->name);
7264 			features &= ~feature;
7265 		}
7266 	}
7267 
7268 	return features;
7269 }
7270 
netdev_sync_lower_features(struct net_device * upper,struct net_device * lower,netdev_features_t features)7271 static void netdev_sync_lower_features(struct net_device *upper,
7272 	struct net_device *lower, netdev_features_t features)
7273 {
7274 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7275 	netdev_features_t feature;
7276 	int feature_bit;
7277 
7278 	for_each_netdev_feature(upper_disables, feature_bit) {
7279 		feature = __NETIF_F_BIT(feature_bit);
7280 		if (!(features & feature) && (lower->features & feature)) {
7281 			netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
7282 				   &feature, lower->name);
7283 			lower->wanted_features &= ~feature;
7284 			netdev_update_features(lower);
7285 
7286 			if (unlikely(lower->features & feature))
7287 				netdev_WARN(upper, "failed to disable %pNF on %s!\n",
7288 					    &feature, lower->name);
7289 		}
7290 	}
7291 }
7292 
netdev_fix_features(struct net_device * dev,netdev_features_t features)7293 static netdev_features_t netdev_fix_features(struct net_device *dev,
7294 	netdev_features_t features)
7295 {
7296 	/* Fix illegal checksum combinations */
7297 	if ((features & NETIF_F_HW_CSUM) &&
7298 	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
7299 		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
7300 		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
7301 	}
7302 
7303 	/* TSO requires that SG is present as well. */
7304 	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
7305 		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
7306 		features &= ~NETIF_F_ALL_TSO;
7307 	}
7308 
7309 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
7310 					!(features & NETIF_F_IP_CSUM)) {
7311 		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
7312 		features &= ~NETIF_F_TSO;
7313 		features &= ~NETIF_F_TSO_ECN;
7314 	}
7315 
7316 	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
7317 					 !(features & NETIF_F_IPV6_CSUM)) {
7318 		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
7319 		features &= ~NETIF_F_TSO6;
7320 	}
7321 
7322 	/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
7323 	if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
7324 		features &= ~NETIF_F_TSO_MANGLEID;
7325 
7326 	/* TSO ECN requires that TSO is present as well. */
7327 	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
7328 		features &= ~NETIF_F_TSO_ECN;
7329 
7330 	/* Software GSO depends on SG. */
7331 	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
7332 		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
7333 		features &= ~NETIF_F_GSO;
7334 	}
7335 
7336 	/* GSO partial features require GSO partial be set */
7337 	if ((features & dev->gso_partial_features) &&
7338 	    !(features & NETIF_F_GSO_PARTIAL)) {
7339 		netdev_dbg(dev,
7340 			   "Dropping partially supported GSO features since no GSO partial.\n");
7341 		features &= ~dev->gso_partial_features;
7342 	}
7343 
7344 	return features;
7345 }
7346 
__netdev_update_features(struct net_device * dev)7347 int __netdev_update_features(struct net_device *dev)
7348 {
7349 	struct net_device *upper, *lower;
7350 	netdev_features_t features;
7351 	struct list_head *iter;
7352 	int err = -1;
7353 
7354 	ASSERT_RTNL();
7355 
7356 	features = netdev_get_wanted_features(dev);
7357 
7358 	if (dev->netdev_ops->ndo_fix_features)
7359 		features = dev->netdev_ops->ndo_fix_features(dev, features);
7360 
7361 	/* driver might be less strict about feature dependencies */
7362 	features = netdev_fix_features(dev, features);
7363 
7364 	/* some features can't be enabled if they're off an an upper device */
7365 	netdev_for_each_upper_dev_rcu(dev, upper, iter)
7366 		features = netdev_sync_upper_features(dev, upper, features);
7367 
7368 	if (dev->features == features)
7369 		goto sync_lower;
7370 
7371 	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7372 		&dev->features, &features);
7373 
7374 	if (dev->netdev_ops->ndo_set_features)
7375 		err = dev->netdev_ops->ndo_set_features(dev, features);
7376 	else
7377 		err = 0;
7378 
7379 	if (unlikely(err < 0)) {
7380 		netdev_err(dev,
7381 			"set_features() failed (%d); wanted %pNF, left %pNF\n",
7382 			err, &features, &dev->features);
7383 		/* return non-0 since some features might have changed and
7384 		 * it's better to fire a spurious notification than miss it
7385 		 */
7386 		return -1;
7387 	}
7388 
7389 sync_lower:
7390 	/* some features must be disabled on lower devices when disabled
7391 	 * on an upper device (think: bonding master or bridge)
7392 	 */
7393 	netdev_for_each_lower_dev(dev, lower, iter)
7394 		netdev_sync_lower_features(dev, lower, features);
7395 
7396 	if (!err) {
7397 		netdev_features_t diff = features ^ dev->features;
7398 
7399 		if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
7400 			/* udp_tunnel_{get,drop}_rx_info both need
7401 			 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
7402 			 * device, or they won't do anything.
7403 			 * Thus we need to update dev->features
7404 			 * *before* calling udp_tunnel_get_rx_info,
7405 			 * but *after* calling udp_tunnel_drop_rx_info.
7406 			 */
7407 			if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
7408 				dev->features = features;
7409 				udp_tunnel_get_rx_info(dev);
7410 			} else {
7411 				udp_tunnel_drop_rx_info(dev);
7412 			}
7413 		}
7414 
7415 		dev->features = features;
7416 	}
7417 
7418 	return err < 0 ? 0 : 1;
7419 }
7420 
7421 /**
7422  *	netdev_update_features - recalculate device features
7423  *	@dev: the device to check
7424  *
7425  *	Recalculate dev->features set and send notifications if it
7426  *	has changed. Should be called after driver or hardware dependent
7427  *	conditions might have changed that influence the features.
7428  */
netdev_update_features(struct net_device * dev)7429 void netdev_update_features(struct net_device *dev)
7430 {
7431 	if (__netdev_update_features(dev))
7432 		netdev_features_change(dev);
7433 }
7434 EXPORT_SYMBOL(netdev_update_features);
7435 
7436 /**
7437  *	netdev_change_features - recalculate device features
7438  *	@dev: the device to check
7439  *
7440  *	Recalculate dev->features set and send notifications even
7441  *	if they have not changed. Should be called instead of
7442  *	netdev_update_features() if also dev->vlan_features might
7443  *	have changed to allow the changes to be propagated to stacked
7444  *	VLAN devices.
7445  */
netdev_change_features(struct net_device * dev)7446 void netdev_change_features(struct net_device *dev)
7447 {
7448 	__netdev_update_features(dev);
7449 	netdev_features_change(dev);
7450 }
7451 EXPORT_SYMBOL(netdev_change_features);
7452 
7453 /**
7454  *	netif_stacked_transfer_operstate -	transfer operstate
7455  *	@rootdev: the root or lower level device to transfer state from
7456  *	@dev: the device to transfer operstate to
7457  *
7458  *	Transfer operational state from root to device. This is normally
7459  *	called when a stacking relationship exists between the root
7460  *	device and the device(a leaf device).
7461  */
netif_stacked_transfer_operstate(const struct net_device * rootdev,struct net_device * dev)7462 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7463 					struct net_device *dev)
7464 {
7465 	if (rootdev->operstate == IF_OPER_DORMANT)
7466 		netif_dormant_on(dev);
7467 	else
7468 		netif_dormant_off(dev);
7469 
7470 	if (netif_carrier_ok(rootdev))
7471 		netif_carrier_on(dev);
7472 	else
7473 		netif_carrier_off(dev);
7474 }
7475 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7476 
7477 #ifdef CONFIG_SYSFS
netif_alloc_rx_queues(struct net_device * dev)7478 static int netif_alloc_rx_queues(struct net_device *dev)
7479 {
7480 	unsigned int i, count = dev->num_rx_queues;
7481 	struct netdev_rx_queue *rx;
7482 	size_t sz = count * sizeof(*rx);
7483 
7484 	BUG_ON(count < 1);
7485 
7486 	rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
7487 	if (!rx)
7488 		return -ENOMEM;
7489 
7490 	dev->_rx = rx;
7491 
7492 	for (i = 0; i < count; i++)
7493 		rx[i].dev = dev;
7494 	return 0;
7495 }
7496 #endif
7497 
netdev_init_one_queue(struct net_device * dev,struct netdev_queue * queue,void * _unused)7498 static void netdev_init_one_queue(struct net_device *dev,
7499 				  struct netdev_queue *queue, void *_unused)
7500 {
7501 	/* Initialize queue lock */
7502 	spin_lock_init(&queue->_xmit_lock);
7503 	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7504 	queue->xmit_lock_owner = -1;
7505 	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
7506 	queue->dev = dev;
7507 #ifdef CONFIG_BQL
7508 	dql_init(&queue->dql, HZ);
7509 #endif
7510 }
7511 
netif_free_tx_queues(struct net_device * dev)7512 static void netif_free_tx_queues(struct net_device *dev)
7513 {
7514 	kvfree(dev->_tx);
7515 }
7516 
netif_alloc_netdev_queues(struct net_device * dev)7517 static int netif_alloc_netdev_queues(struct net_device *dev)
7518 {
7519 	unsigned int count = dev->num_tx_queues;
7520 	struct netdev_queue *tx;
7521 	size_t sz = count * sizeof(*tx);
7522 
7523 	if (count < 1 || count > 0xffff)
7524 		return -EINVAL;
7525 
7526 	tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
7527 	if (!tx)
7528 		return -ENOMEM;
7529 
7530 	dev->_tx = tx;
7531 
7532 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7533 	spin_lock_init(&dev->tx_global_lock);
7534 
7535 	return 0;
7536 }
7537 
netif_tx_stop_all_queues(struct net_device * dev)7538 void netif_tx_stop_all_queues(struct net_device *dev)
7539 {
7540 	unsigned int i;
7541 
7542 	for (i = 0; i < dev->num_tx_queues; i++) {
7543 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
7544 
7545 		netif_tx_stop_queue(txq);
7546 	}
7547 }
7548 EXPORT_SYMBOL(netif_tx_stop_all_queues);
7549 
7550 /**
7551  *	register_netdevice	- register a network device
7552  *	@dev: device to register
7553  *
7554  *	Take a completed network device structure and add it to the kernel
7555  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7556  *	chain. 0 is returned on success. A negative errno code is returned
7557  *	on a failure to set up the device, or if the name is a duplicate.
7558  *
7559  *	Callers must hold the rtnl semaphore. You may want
7560  *	register_netdev() instead of this.
7561  *
7562  *	BUGS:
7563  *	The locking appears insufficient to guarantee two parallel registers
7564  *	will not get the same name.
7565  */
7566 
register_netdevice(struct net_device * dev)7567 int register_netdevice(struct net_device *dev)
7568 {
7569 	int ret;
7570 	struct net *net = dev_net(dev);
7571 
7572 	BUG_ON(dev_boot_phase);
7573 	ASSERT_RTNL();
7574 
7575 	might_sleep();
7576 
7577 	/* When net_device's are persistent, this will be fatal. */
7578 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
7579 	BUG_ON(!net);
7580 
7581 	spin_lock_init(&dev->addr_list_lock);
7582 	netdev_set_addr_lockdep_class(dev);
7583 
7584 	ret = dev_get_valid_name(net, dev, dev->name);
7585 	if (ret < 0)
7586 		goto out;
7587 
7588 	/* Init, if this function is available */
7589 	if (dev->netdev_ops->ndo_init) {
7590 		ret = dev->netdev_ops->ndo_init(dev);
7591 		if (ret) {
7592 			if (ret > 0)
7593 				ret = -EIO;
7594 			goto out;
7595 		}
7596 	}
7597 
7598 	if (((dev->hw_features | dev->features) &
7599 	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
7600 	    (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7601 	     !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7602 		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7603 		ret = -EINVAL;
7604 		goto err_uninit;
7605 	}
7606 
7607 	ret = -EBUSY;
7608 	if (!dev->ifindex)
7609 		dev->ifindex = dev_new_index(net);
7610 	else if (__dev_get_by_index(net, dev->ifindex))
7611 		goto err_uninit;
7612 
7613 	/* Transfer changeable features to wanted_features and enable
7614 	 * software offloads (GSO and GRO).
7615 	 */
7616 	dev->hw_features |= NETIF_F_SOFT_FEATURES;
7617 	dev->features |= NETIF_F_SOFT_FEATURES;
7618 
7619 	if (dev->netdev_ops->ndo_udp_tunnel_add) {
7620 		dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7621 		dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7622 	}
7623 
7624 	dev->wanted_features = dev->features & dev->hw_features;
7625 
7626 	if (!(dev->flags & IFF_LOOPBACK))
7627 		dev->hw_features |= NETIF_F_NOCACHE_COPY;
7628 
7629 	/* If IPv4 TCP segmentation offload is supported we should also
7630 	 * allow the device to enable segmenting the frame with the option
7631 	 * of ignoring a static IP ID value.  This doesn't enable the
7632 	 * feature itself but allows the user to enable it later.
7633 	 */
7634 	if (dev->hw_features & NETIF_F_TSO)
7635 		dev->hw_features |= NETIF_F_TSO_MANGLEID;
7636 	if (dev->vlan_features & NETIF_F_TSO)
7637 		dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7638 	if (dev->mpls_features & NETIF_F_TSO)
7639 		dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7640 	if (dev->hw_enc_features & NETIF_F_TSO)
7641 		dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
7642 
7643 	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
7644 	 */
7645 	dev->vlan_features |= NETIF_F_HIGHDMA;
7646 
7647 	/* Make NETIF_F_SG inheritable to tunnel devices.
7648 	 */
7649 	dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
7650 
7651 	/* Make NETIF_F_SG inheritable to MPLS.
7652 	 */
7653 	dev->mpls_features |= NETIF_F_SG;
7654 
7655 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7656 	ret = notifier_to_errno(ret);
7657 	if (ret)
7658 		goto err_uninit;
7659 
7660 	ret = netdev_register_kobject(dev);
7661 	if (ret) {
7662 		dev->reg_state = NETREG_UNREGISTERED;
7663 		goto err_uninit;
7664 	}
7665 	dev->reg_state = NETREG_REGISTERED;
7666 
7667 	__netdev_update_features(dev);
7668 
7669 	/*
7670 	 *	Default initial state at registry is that the
7671 	 *	device is present.
7672 	 */
7673 
7674 	set_bit(__LINK_STATE_PRESENT, &dev->state);
7675 
7676 	linkwatch_init_dev(dev);
7677 
7678 	dev_init_scheduler(dev);
7679 	dev_hold(dev);
7680 	list_netdevice(dev);
7681 	add_device_randomness(dev->dev_addr, dev->addr_len);
7682 
7683 	/* If the device has permanent device address, driver should
7684 	 * set dev_addr and also addr_assign_type should be set to
7685 	 * NET_ADDR_PERM (default value).
7686 	 */
7687 	if (dev->addr_assign_type == NET_ADDR_PERM)
7688 		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7689 
7690 	/* Notify protocols, that a new device appeared. */
7691 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7692 	ret = notifier_to_errno(ret);
7693 	if (ret) {
7694 		rollback_registered(dev);
7695 		rcu_barrier();
7696 
7697 		dev->reg_state = NETREG_UNREGISTERED;
7698 	}
7699 	/*
7700 	 *	Prevent userspace races by waiting until the network
7701 	 *	device is fully setup before sending notifications.
7702 	 */
7703 	if (!dev->rtnl_link_ops ||
7704 	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7705 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7706 
7707 out:
7708 	return ret;
7709 
7710 err_uninit:
7711 	if (dev->netdev_ops->ndo_uninit)
7712 		dev->netdev_ops->ndo_uninit(dev);
7713 	if (dev->priv_destructor)
7714 		dev->priv_destructor(dev);
7715 	goto out;
7716 }
7717 EXPORT_SYMBOL(register_netdevice);
7718 
7719 /**
7720  *	init_dummy_netdev	- init a dummy network device for NAPI
7721  *	@dev: device to init
7722  *
7723  *	This takes a network device structure and initialize the minimum
7724  *	amount of fields so it can be used to schedule NAPI polls without
7725  *	registering a full blown interface. This is to be used by drivers
7726  *	that need to tie several hardware interfaces to a single NAPI
7727  *	poll scheduler due to HW limitations.
7728  */
init_dummy_netdev(struct net_device * dev)7729 int init_dummy_netdev(struct net_device *dev)
7730 {
7731 	/* Clear everything. Note we don't initialize spinlocks
7732 	 * are they aren't supposed to be taken by any of the
7733 	 * NAPI code and this dummy netdev is supposed to be
7734 	 * only ever used for NAPI polls
7735 	 */
7736 	memset(dev, 0, sizeof(struct net_device));
7737 
7738 	/* make sure we BUG if trying to hit standard
7739 	 * register/unregister code path
7740 	 */
7741 	dev->reg_state = NETREG_DUMMY;
7742 
7743 	/* NAPI wants this */
7744 	INIT_LIST_HEAD(&dev->napi_list);
7745 
7746 	/* a dummy interface is started by default */
7747 	set_bit(__LINK_STATE_PRESENT, &dev->state);
7748 	set_bit(__LINK_STATE_START, &dev->state);
7749 
7750 	/* napi_busy_loop stats accounting wants this */
7751 	dev_net_set(dev, &init_net);
7752 
7753 	/* Note : We dont allocate pcpu_refcnt for dummy devices,
7754 	 * because users of this 'device' dont need to change
7755 	 * its refcount.
7756 	 */
7757 
7758 	return 0;
7759 }
7760 EXPORT_SYMBOL_GPL(init_dummy_netdev);
7761 
7762 
dev_validate_mtu(struct net_device * dev,int new_mtu)7763 int dev_validate_mtu(struct net_device *dev, int new_mtu)
7764 {
7765 	/* MTU must be positive, and in range */
7766 	if (new_mtu < 0 || new_mtu < dev->min_mtu) {
7767 		net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
7768 				    dev->name, new_mtu, dev->min_mtu);
7769 		return -EINVAL;
7770 	}
7771 
7772 	if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
7773 		net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
7774 				    dev->name, new_mtu, dev->max_mtu);
7775 		return -EINVAL;
7776 	}
7777 	return 0;
7778 }
7779 
7780 /**
7781  *	register_netdev	- register a network device
7782  *	@dev: device to register
7783  *
7784  *	Take a completed network device structure and add it to the kernel
7785  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7786  *	chain. 0 is returned on success. A negative errno code is returned
7787  *	on a failure to set up the device, or if the name is a duplicate.
7788  *
7789  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
7790  *	and expands the device name if you passed a format string to
7791  *	alloc_netdev.
7792  */
register_netdev(struct net_device * dev)7793 int register_netdev(struct net_device *dev)
7794 {
7795 	int err;
7796 
7797 	rtnl_lock();
7798 	err = register_netdevice(dev);
7799 	rtnl_unlock();
7800 	return err;
7801 }
7802 EXPORT_SYMBOL(register_netdev);
7803 
netdev_refcnt_read(const struct net_device * dev)7804 int netdev_refcnt_read(const struct net_device *dev)
7805 {
7806 	int i, refcnt = 0;
7807 
7808 	for_each_possible_cpu(i)
7809 		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7810 	return refcnt;
7811 }
7812 EXPORT_SYMBOL(netdev_refcnt_read);
7813 
7814 /**
7815  * netdev_wait_allrefs - wait until all references are gone.
7816  * @dev: target net_device
7817  *
7818  * This is called when unregistering network devices.
7819  *
7820  * Any protocol or device that holds a reference should register
7821  * for netdevice notification, and cleanup and put back the
7822  * reference if they receive an UNREGISTER event.
7823  * We can get stuck here if buggy protocols don't correctly
7824  * call dev_put.
7825  */
netdev_wait_allrefs(struct net_device * dev)7826 static void netdev_wait_allrefs(struct net_device *dev)
7827 {
7828 	unsigned long rebroadcast_time, warning_time;
7829 	int refcnt;
7830 
7831 	linkwatch_forget_dev(dev);
7832 
7833 	rebroadcast_time = warning_time = jiffies;
7834 	refcnt = netdev_refcnt_read(dev);
7835 
7836 	while (refcnt != 0) {
7837 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7838 			rtnl_lock();
7839 
7840 			/* Rebroadcast unregister notification */
7841 			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7842 
7843 			__rtnl_unlock();
7844 			rcu_barrier();
7845 			rtnl_lock();
7846 
7847 			call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7848 			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7849 				     &dev->state)) {
7850 				/* We must not have linkwatch events
7851 				 * pending on unregister. If this
7852 				 * happens, we simply run the queue
7853 				 * unscheduled, resulting in a noop
7854 				 * for this device.
7855 				 */
7856 				linkwatch_run_queue();
7857 			}
7858 
7859 			__rtnl_unlock();
7860 
7861 			rebroadcast_time = jiffies;
7862 		}
7863 
7864 		msleep(250);
7865 
7866 		refcnt = netdev_refcnt_read(dev);
7867 
7868 		if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) {
7869 			pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7870 				 dev->name, refcnt);
7871 			warning_time = jiffies;
7872 		}
7873 	}
7874 }
7875 
7876 /* The sequence is:
7877  *
7878  *	rtnl_lock();
7879  *	...
7880  *	register_netdevice(x1);
7881  *	register_netdevice(x2);
7882  *	...
7883  *	unregister_netdevice(y1);
7884  *	unregister_netdevice(y2);
7885  *      ...
7886  *	rtnl_unlock();
7887  *	free_netdev(y1);
7888  *	free_netdev(y2);
7889  *
7890  * We are invoked by rtnl_unlock().
7891  * This allows us to deal with problems:
7892  * 1) We can delete sysfs objects which invoke hotplug
7893  *    without deadlocking with linkwatch via keventd.
7894  * 2) Since we run with the RTNL semaphore not held, we can sleep
7895  *    safely in order to wait for the netdev refcnt to drop to zero.
7896  *
7897  * We must not return until all unregister events added during
7898  * the interval the lock was held have been completed.
7899  */
netdev_run_todo(void)7900 void netdev_run_todo(void)
7901 {
7902 	struct list_head list;
7903 
7904 	/* Snapshot list, allow later requests */
7905 	list_replace_init(&net_todo_list, &list);
7906 
7907 	__rtnl_unlock();
7908 
7909 
7910 	/* Wait for rcu callbacks to finish before next phase */
7911 	if (!list_empty(&list))
7912 		rcu_barrier();
7913 
7914 	while (!list_empty(&list)) {
7915 		struct net_device *dev
7916 			= list_first_entry(&list, struct net_device, todo_list);
7917 		list_del(&dev->todo_list);
7918 
7919 		rtnl_lock();
7920 		call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7921 		__rtnl_unlock();
7922 
7923 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7924 			pr_err("network todo '%s' but state %d\n",
7925 			       dev->name, dev->reg_state);
7926 			dump_stack();
7927 			continue;
7928 		}
7929 
7930 		dev->reg_state = NETREG_UNREGISTERED;
7931 
7932 		netdev_wait_allrefs(dev);
7933 
7934 		/* paranoia */
7935 		BUG_ON(netdev_refcnt_read(dev));
7936 		BUG_ON(!list_empty(&dev->ptype_all));
7937 		BUG_ON(!list_empty(&dev->ptype_specific));
7938 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
7939 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7940 		WARN_ON(dev->dn_ptr);
7941 
7942 		if (dev->priv_destructor)
7943 			dev->priv_destructor(dev);
7944 		if (dev->needs_free_netdev)
7945 			free_netdev(dev);
7946 
7947 		/* Report a network device has been unregistered */
7948 		rtnl_lock();
7949 		dev_net(dev)->dev_unreg_count--;
7950 		__rtnl_unlock();
7951 		wake_up(&netdev_unregistering_wq);
7952 
7953 		/* Free network device */
7954 		kobject_put(&dev->dev.kobj);
7955 	}
7956 }
7957 
7958 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7959  * all the same fields in the same order as net_device_stats, with only
7960  * the type differing, but rtnl_link_stats64 may have additional fields
7961  * at the end for newer counters.
7962  */
netdev_stats_to_stats64(struct rtnl_link_stats64 * stats64,const struct net_device_stats * netdev_stats)7963 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7964 			     const struct net_device_stats *netdev_stats)
7965 {
7966 #if BITS_PER_LONG == 64
7967 	BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
7968 	memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
7969 	/* zero out counters that only exist in rtnl_link_stats64 */
7970 	memset((char *)stats64 + sizeof(*netdev_stats), 0,
7971 	       sizeof(*stats64) - sizeof(*netdev_stats));
7972 #else
7973 	size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
7974 	const unsigned long *src = (const unsigned long *)netdev_stats;
7975 	u64 *dst = (u64 *)stats64;
7976 
7977 	BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
7978 	for (i = 0; i < n; i++)
7979 		dst[i] = src[i];
7980 	/* zero out counters that only exist in rtnl_link_stats64 */
7981 	memset((char *)stats64 + n * sizeof(u64), 0,
7982 	       sizeof(*stats64) - n * sizeof(u64));
7983 #endif
7984 }
7985 EXPORT_SYMBOL(netdev_stats_to_stats64);
7986 
7987 /**
7988  *	dev_get_stats	- get network device statistics
7989  *	@dev: device to get statistics from
7990  *	@storage: place to store stats
7991  *
7992  *	Get network statistics from device. Return @storage.
7993  *	The device driver may provide its own method by setting
7994  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7995  *	otherwise the internal statistics structure is used.
7996  */
dev_get_stats(struct net_device * dev,struct rtnl_link_stats64 * storage)7997 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7998 					struct rtnl_link_stats64 *storage)
7999 {
8000 	const struct net_device_ops *ops = dev->netdev_ops;
8001 
8002 	if (ops->ndo_get_stats64) {
8003 		memset(storage, 0, sizeof(*storage));
8004 		ops->ndo_get_stats64(dev, storage);
8005 	} else if (ops->ndo_get_stats) {
8006 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
8007 	} else {
8008 		netdev_stats_to_stats64(storage, &dev->stats);
8009 	}
8010 	storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
8011 	storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
8012 	storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
8013 	return storage;
8014 }
8015 EXPORT_SYMBOL(dev_get_stats);
8016 
dev_ingress_queue_create(struct net_device * dev)8017 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
8018 {
8019 	struct netdev_queue *queue = dev_ingress_queue(dev);
8020 
8021 #ifdef CONFIG_NET_CLS_ACT
8022 	if (queue)
8023 		return queue;
8024 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
8025 	if (!queue)
8026 		return NULL;
8027 	netdev_init_one_queue(dev, queue, NULL);
8028 	RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
8029 	queue->qdisc_sleeping = &noop_qdisc;
8030 	rcu_assign_pointer(dev->ingress_queue, queue);
8031 #endif
8032 	return queue;
8033 }
8034 
8035 static const struct ethtool_ops default_ethtool_ops;
8036 
netdev_set_default_ethtool_ops(struct net_device * dev,const struct ethtool_ops * ops)8037 void netdev_set_default_ethtool_ops(struct net_device *dev,
8038 				    const struct ethtool_ops *ops)
8039 {
8040 	if (dev->ethtool_ops == &default_ethtool_ops)
8041 		dev->ethtool_ops = ops;
8042 }
8043 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
8044 
netdev_freemem(struct net_device * dev)8045 void netdev_freemem(struct net_device *dev)
8046 {
8047 	char *addr = (char *)dev - dev->padded;
8048 
8049 	kvfree(addr);
8050 }
8051 
8052 /**
8053  * alloc_netdev_mqs - allocate network device
8054  * @sizeof_priv: size of private data to allocate space for
8055  * @name: device name format string
8056  * @name_assign_type: origin of device name
8057  * @setup: callback to initialize device
8058  * @txqs: the number of TX subqueues to allocate
8059  * @rxqs: the number of RX subqueues to allocate
8060  *
8061  * Allocates a struct net_device with private data area for driver use
8062  * and performs basic initialization.  Also allocates subqueue structs
8063  * for each queue on the device.
8064  */
alloc_netdev_mqs(int sizeof_priv,const char * name,unsigned char name_assign_type,void (* setup)(struct net_device *),unsigned int txqs,unsigned int rxqs)8065 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
8066 		unsigned char name_assign_type,
8067 		void (*setup)(struct net_device *),
8068 		unsigned int txqs, unsigned int rxqs)
8069 {
8070 	struct net_device *dev;
8071 	size_t alloc_size;
8072 	struct net_device *p;
8073 
8074 	BUG_ON(strlen(name) >= sizeof(dev->name));
8075 
8076 	if (txqs < 1) {
8077 		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
8078 		return NULL;
8079 	}
8080 
8081 #ifdef CONFIG_SYSFS
8082 	if (rxqs < 1) {
8083 		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
8084 		return NULL;
8085 	}
8086 #endif
8087 
8088 	alloc_size = sizeof(struct net_device);
8089 	if (sizeof_priv) {
8090 		/* ensure 32-byte alignment of private area */
8091 		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
8092 		alloc_size += sizeof_priv;
8093 	}
8094 	/* ensure 32-byte alignment of whole construct */
8095 	alloc_size += NETDEV_ALIGN - 1;
8096 
8097 	p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
8098 	if (!p)
8099 		return NULL;
8100 
8101 	dev = PTR_ALIGN(p, NETDEV_ALIGN);
8102 	dev->padded = (char *)dev - (char *)p;
8103 
8104 	dev->pcpu_refcnt = alloc_percpu(int);
8105 	if (!dev->pcpu_refcnt)
8106 		goto free_dev;
8107 
8108 	if (dev_addr_init(dev))
8109 		goto free_pcpu;
8110 
8111 	dev_mc_init(dev);
8112 	dev_uc_init(dev);
8113 
8114 	dev_net_set(dev, &init_net);
8115 
8116 	dev->gso_max_size = GSO_MAX_SIZE;
8117 	dev->gso_max_segs = GSO_MAX_SEGS;
8118 
8119 	INIT_LIST_HEAD(&dev->napi_list);
8120 	INIT_LIST_HEAD(&dev->unreg_list);
8121 	INIT_LIST_HEAD(&dev->close_list);
8122 	INIT_LIST_HEAD(&dev->link_watch_list);
8123 	INIT_LIST_HEAD(&dev->adj_list.upper);
8124 	INIT_LIST_HEAD(&dev->adj_list.lower);
8125 	INIT_LIST_HEAD(&dev->ptype_all);
8126 	INIT_LIST_HEAD(&dev->ptype_specific);
8127 #ifdef CONFIG_NET_SCHED
8128 	hash_init(dev->qdisc_hash);
8129 #endif
8130 	dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
8131 	setup(dev);
8132 
8133 	if (!dev->tx_queue_len) {
8134 		dev->priv_flags |= IFF_NO_QUEUE;
8135 		dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
8136 	}
8137 
8138 	dev->num_tx_queues = txqs;
8139 	dev->real_num_tx_queues = txqs;
8140 	if (netif_alloc_netdev_queues(dev))
8141 		goto free_all;
8142 
8143 #ifdef CONFIG_SYSFS
8144 	dev->num_rx_queues = rxqs;
8145 	dev->real_num_rx_queues = rxqs;
8146 	if (netif_alloc_rx_queues(dev))
8147 		goto free_all;
8148 #endif
8149 
8150 	strcpy(dev->name, name);
8151 	dev->name_assign_type = name_assign_type;
8152 	dev->group = INIT_NETDEV_GROUP;
8153 	if (!dev->ethtool_ops)
8154 		dev->ethtool_ops = &default_ethtool_ops;
8155 
8156 	nf_hook_ingress_init(dev);
8157 
8158 	return dev;
8159 
8160 free_all:
8161 	free_netdev(dev);
8162 	return NULL;
8163 
8164 free_pcpu:
8165 	free_percpu(dev->pcpu_refcnt);
8166 free_dev:
8167 	netdev_freemem(dev);
8168 	return NULL;
8169 }
8170 EXPORT_SYMBOL(alloc_netdev_mqs);
8171 
8172 /**
8173  * free_netdev - free network device
8174  * @dev: device
8175  *
8176  * This function does the last stage of destroying an allocated device
8177  * interface. The reference to the device object is released. If this
8178  * is the last reference then it will be freed.Must be called in process
8179  * context.
8180  */
free_netdev(struct net_device * dev)8181 void free_netdev(struct net_device *dev)
8182 {
8183 	struct napi_struct *p, *n;
8184 	struct bpf_prog *prog;
8185 
8186 	might_sleep();
8187 	netif_free_tx_queues(dev);
8188 #ifdef CONFIG_SYSFS
8189 	kvfree(dev->_rx);
8190 #endif
8191 
8192 	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
8193 
8194 	/* Flush device addresses */
8195 	dev_addr_flush(dev);
8196 
8197 	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
8198 		netif_napi_del(p);
8199 
8200 	free_percpu(dev->pcpu_refcnt);
8201 	dev->pcpu_refcnt = NULL;
8202 
8203 	prog = rcu_dereference_protected(dev->xdp_prog, 1);
8204 	if (prog) {
8205 		bpf_prog_put(prog);
8206 		static_key_slow_dec(&generic_xdp_needed);
8207 	}
8208 
8209 	/*  Compatibility with error handling in drivers */
8210 	if (dev->reg_state == NETREG_UNINITIALIZED) {
8211 		netdev_freemem(dev);
8212 		return;
8213 	}
8214 
8215 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
8216 	dev->reg_state = NETREG_RELEASED;
8217 
8218 	/* will free via device release */
8219 	put_device(&dev->dev);
8220 }
8221 EXPORT_SYMBOL(free_netdev);
8222 
8223 /**
8224  *	synchronize_net -  Synchronize with packet receive processing
8225  *
8226  *	Wait for packets currently being received to be done.
8227  *	Does not block later packets from starting.
8228  */
synchronize_net(void)8229 void synchronize_net(void)
8230 {
8231 	might_sleep();
8232 	if (rtnl_is_locked())
8233 		synchronize_rcu_expedited();
8234 	else
8235 		synchronize_rcu();
8236 }
8237 EXPORT_SYMBOL(synchronize_net);
8238 
8239 /**
8240  *	unregister_netdevice_queue - remove device from the kernel
8241  *	@dev: device
8242  *	@head: list
8243  *
8244  *	This function shuts down a device interface and removes it
8245  *	from the kernel tables.
8246  *	If head not NULL, device is queued to be unregistered later.
8247  *
8248  *	Callers must hold the rtnl semaphore.  You may want
8249  *	unregister_netdev() instead of this.
8250  */
8251 
unregister_netdevice_queue(struct net_device * dev,struct list_head * head)8252 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
8253 {
8254 	ASSERT_RTNL();
8255 
8256 	if (head) {
8257 		list_move_tail(&dev->unreg_list, head);
8258 	} else {
8259 		rollback_registered(dev);
8260 		/* Finish processing unregister after unlock */
8261 		net_set_todo(dev);
8262 	}
8263 }
8264 EXPORT_SYMBOL(unregister_netdevice_queue);
8265 
8266 /**
8267  *	unregister_netdevice_many - unregister many devices
8268  *	@head: list of devices
8269  *
8270  *  Note: As most callers use a stack allocated list_head,
8271  *  we force a list_del() to make sure stack wont be corrupted later.
8272  */
unregister_netdevice_many(struct list_head * head)8273 void unregister_netdevice_many(struct list_head *head)
8274 {
8275 	struct net_device *dev;
8276 
8277 	if (!list_empty(head)) {
8278 		rollback_registered_many(head);
8279 		list_for_each_entry(dev, head, unreg_list)
8280 			net_set_todo(dev);
8281 		list_del(head);
8282 	}
8283 }
8284 EXPORT_SYMBOL(unregister_netdevice_many);
8285 
8286 /**
8287  *	unregister_netdev - remove device from the kernel
8288  *	@dev: device
8289  *
8290  *	This function shuts down a device interface and removes it
8291  *	from the kernel tables.
8292  *
8293  *	This is just a wrapper for unregister_netdevice that takes
8294  *	the rtnl semaphore.  In general you want to use this and not
8295  *	unregister_netdevice.
8296  */
unregister_netdev(struct net_device * dev)8297 void unregister_netdev(struct net_device *dev)
8298 {
8299 	rtnl_lock();
8300 	unregister_netdevice(dev);
8301 	rtnl_unlock();
8302 }
8303 EXPORT_SYMBOL(unregister_netdev);
8304 
8305 /**
8306  *	dev_change_net_namespace - move device to different nethost namespace
8307  *	@dev: device
8308  *	@net: network namespace
8309  *	@pat: If not NULL name pattern to try if the current device name
8310  *	      is already taken in the destination network namespace.
8311  *
8312  *	This function shuts down a device interface and moves it
8313  *	to a new network namespace. On success 0 is returned, on
8314  *	a failure a netagive errno code is returned.
8315  *
8316  *	Callers must hold the rtnl semaphore.
8317  */
8318 
dev_change_net_namespace(struct net_device * dev,struct net * net,const char * pat)8319 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
8320 {
8321 	int err;
8322 
8323 	ASSERT_RTNL();
8324 
8325 	/* Don't allow namespace local devices to be moved. */
8326 	err = -EINVAL;
8327 	if (dev->features & NETIF_F_NETNS_LOCAL)
8328 		goto out;
8329 
8330 	/* Ensure the device has been registrered */
8331 	if (dev->reg_state != NETREG_REGISTERED)
8332 		goto out;
8333 
8334 	/* Get out if there is nothing todo */
8335 	err = 0;
8336 	if (net_eq(dev_net(dev), net))
8337 		goto out;
8338 
8339 	/* Pick the destination device name, and ensure
8340 	 * we can use it in the destination network namespace.
8341 	 */
8342 	err = -EEXIST;
8343 	if (__dev_get_by_name(net, dev->name)) {
8344 		/* We get here if we can't use the current device name */
8345 		if (!pat)
8346 			goto out;
8347 		err = dev_get_valid_name(net, dev, pat);
8348 		if (err < 0)
8349 			goto out;
8350 	}
8351 
8352 	/*
8353 	 * And now a mini version of register_netdevice unregister_netdevice.
8354 	 */
8355 
8356 	/* If device is running close it first. */
8357 	dev_close(dev);
8358 
8359 	/* And unlink it from device chain */
8360 	unlist_netdevice(dev);
8361 
8362 	synchronize_net();
8363 
8364 	/* Shutdown queueing discipline. */
8365 	dev_shutdown(dev);
8366 
8367 	/* Notify protocols, that we are about to destroy
8368 	 * this device. They should clean all the things.
8369 	 *
8370 	 * Note that dev->reg_state stays at NETREG_REGISTERED.
8371 	 * This is wanted because this way 8021q and macvlan know
8372 	 * the device is just moving and can keep their slaves up.
8373 	 */
8374 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8375 	rcu_barrier();
8376 	call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
8377 	rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
8378 
8379 	/*
8380 	 *	Flush the unicast and multicast chains
8381 	 */
8382 	dev_uc_flush(dev);
8383 	dev_mc_flush(dev);
8384 
8385 	/* Send a netdev-removed uevent to the old namespace */
8386 	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
8387 	netdev_adjacent_del_links(dev);
8388 
8389 	/* Actually switch the network namespace */
8390 	dev_net_set(dev, net);
8391 
8392 	/* If there is an ifindex conflict assign a new one */
8393 	if (__dev_get_by_index(net, dev->ifindex))
8394 		dev->ifindex = dev_new_index(net);
8395 
8396 	/* Send a netdev-add uevent to the new namespace */
8397 	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
8398 	netdev_adjacent_add_links(dev);
8399 
8400 	/* Fixup kobjects */
8401 	err = device_rename(&dev->dev, dev->name);
8402 	WARN_ON(err);
8403 
8404 	/* Add the device back in the hashes */
8405 	list_netdevice(dev);
8406 
8407 	/* Notify protocols, that a new device appeared. */
8408 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
8409 
8410 	/*
8411 	 *	Prevent userspace races by waiting until the network
8412 	 *	device is fully setup before sending notifications.
8413 	 */
8414 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
8415 
8416 	synchronize_net();
8417 	err = 0;
8418 out:
8419 	return err;
8420 }
8421 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
8422 
dev_cpu_dead(unsigned int oldcpu)8423 static int dev_cpu_dead(unsigned int oldcpu)
8424 {
8425 	struct sk_buff **list_skb;
8426 	struct sk_buff *skb;
8427 	unsigned int cpu;
8428 	struct softnet_data *sd, *oldsd, *remsd = NULL;
8429 
8430 	local_irq_disable();
8431 	cpu = smp_processor_id();
8432 	sd = &per_cpu(softnet_data, cpu);
8433 	oldsd = &per_cpu(softnet_data, oldcpu);
8434 
8435 	/* Find end of our completion_queue. */
8436 	list_skb = &sd->completion_queue;
8437 	while (*list_skb)
8438 		list_skb = &(*list_skb)->next;
8439 	/* Append completion queue from offline CPU. */
8440 	*list_skb = oldsd->completion_queue;
8441 	oldsd->completion_queue = NULL;
8442 
8443 	/* Append output queue from offline CPU. */
8444 	if (oldsd->output_queue) {
8445 		*sd->output_queue_tailp = oldsd->output_queue;
8446 		sd->output_queue_tailp = oldsd->output_queue_tailp;
8447 		oldsd->output_queue = NULL;
8448 		oldsd->output_queue_tailp = &oldsd->output_queue;
8449 	}
8450 	/* Append NAPI poll list from offline CPU, with one exception :
8451 	 * process_backlog() must be called by cpu owning percpu backlog.
8452 	 * We properly handle process_queue & input_pkt_queue later.
8453 	 */
8454 	while (!list_empty(&oldsd->poll_list)) {
8455 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8456 							    struct napi_struct,
8457 							    poll_list);
8458 
8459 		list_del_init(&napi->poll_list);
8460 		if (napi->poll == process_backlog)
8461 			napi->state = 0;
8462 		else
8463 			____napi_schedule(sd, napi);
8464 	}
8465 
8466 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
8467 	local_irq_enable();
8468 
8469 #ifdef CONFIG_RPS
8470 	remsd = oldsd->rps_ipi_list;
8471 	oldsd->rps_ipi_list = NULL;
8472 #endif
8473 	/* send out pending IPI's on offline CPU */
8474 	net_rps_send_ipi(remsd);
8475 
8476 	/* Process offline CPU's input_pkt_queue */
8477 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
8478 		netif_rx_ni(skb);
8479 		input_queue_head_incr(oldsd);
8480 	}
8481 	while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
8482 		netif_rx_ni(skb);
8483 		input_queue_head_incr(oldsd);
8484 	}
8485 
8486 	return 0;
8487 }
8488 
8489 /**
8490  *	netdev_increment_features - increment feature set by one
8491  *	@all: current feature set
8492  *	@one: new feature set
8493  *	@mask: mask feature set
8494  *
8495  *	Computes a new feature set after adding a device with feature set
8496  *	@one to the master device with current feature set @all.  Will not
8497  *	enable anything that is off in @mask. Returns the new feature set.
8498  */
netdev_increment_features(netdev_features_t all,netdev_features_t one,netdev_features_t mask)8499 netdev_features_t netdev_increment_features(netdev_features_t all,
8500 	netdev_features_t one, netdev_features_t mask)
8501 {
8502 	if (mask & NETIF_F_HW_CSUM)
8503 		mask |= NETIF_F_CSUM_MASK;
8504 	mask |= NETIF_F_VLAN_CHALLENGED;
8505 
8506 	all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
8507 	all &= one | ~NETIF_F_ALL_FOR_ALL;
8508 
8509 	/* If one device supports hw checksumming, set for all. */
8510 	if (all & NETIF_F_HW_CSUM)
8511 		all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
8512 
8513 	return all;
8514 }
8515 EXPORT_SYMBOL(netdev_increment_features);
8516 
netdev_create_hash(void)8517 static struct hlist_head * __net_init netdev_create_hash(void)
8518 {
8519 	int i;
8520 	struct hlist_head *hash;
8521 
8522 	hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8523 	if (hash != NULL)
8524 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
8525 			INIT_HLIST_HEAD(&hash[i]);
8526 
8527 	return hash;
8528 }
8529 
8530 /* Initialize per network namespace state */
netdev_init(struct net * net)8531 static int __net_init netdev_init(struct net *net)
8532 {
8533 	if (net != &init_net)
8534 		INIT_LIST_HEAD(&net->dev_base_head);
8535 
8536 	net->dev_name_head = netdev_create_hash();
8537 	if (net->dev_name_head == NULL)
8538 		goto err_name;
8539 
8540 	net->dev_index_head = netdev_create_hash();
8541 	if (net->dev_index_head == NULL)
8542 		goto err_idx;
8543 
8544 	return 0;
8545 
8546 err_idx:
8547 	kfree(net->dev_name_head);
8548 err_name:
8549 	return -ENOMEM;
8550 }
8551 
8552 /**
8553  *	netdev_drivername - network driver for the device
8554  *	@dev: network device
8555  *
8556  *	Determine network driver for device.
8557  */
netdev_drivername(const struct net_device * dev)8558 const char *netdev_drivername(const struct net_device *dev)
8559 {
8560 	const struct device_driver *driver;
8561 	const struct device *parent;
8562 	const char *empty = "";
8563 
8564 	parent = dev->dev.parent;
8565 	if (!parent)
8566 		return empty;
8567 
8568 	driver = parent->driver;
8569 	if (driver && driver->name)
8570 		return driver->name;
8571 	return empty;
8572 }
8573 
__netdev_printk(const char * level,const struct net_device * dev,struct va_format * vaf)8574 static void __netdev_printk(const char *level, const struct net_device *dev,
8575 			    struct va_format *vaf)
8576 {
8577 	if (dev && dev->dev.parent) {
8578 		dev_printk_emit(level[1] - '0',
8579 				dev->dev.parent,
8580 				"%s %s %s%s: %pV",
8581 				dev_driver_string(dev->dev.parent),
8582 				dev_name(dev->dev.parent),
8583 				netdev_name(dev), netdev_reg_state(dev),
8584 				vaf);
8585 	} else if (dev) {
8586 		printk("%s%s%s: %pV",
8587 		       level, netdev_name(dev), netdev_reg_state(dev), vaf);
8588 	} else {
8589 		printk("%s(NULL net_device): %pV", level, vaf);
8590 	}
8591 }
8592 
netdev_printk(const char * level,const struct net_device * dev,const char * format,...)8593 void netdev_printk(const char *level, const struct net_device *dev,
8594 		   const char *format, ...)
8595 {
8596 	struct va_format vaf;
8597 	va_list args;
8598 
8599 	va_start(args, format);
8600 
8601 	vaf.fmt = format;
8602 	vaf.va = &args;
8603 
8604 	__netdev_printk(level, dev, &vaf);
8605 
8606 	va_end(args);
8607 }
8608 EXPORT_SYMBOL(netdev_printk);
8609 
8610 #define define_netdev_printk_level(func, level)			\
8611 void func(const struct net_device *dev, const char *fmt, ...)	\
8612 {								\
8613 	struct va_format vaf;					\
8614 	va_list args;						\
8615 								\
8616 	va_start(args, fmt);					\
8617 								\
8618 	vaf.fmt = fmt;						\
8619 	vaf.va = &args;						\
8620 								\
8621 	__netdev_printk(level, dev, &vaf);			\
8622 								\
8623 	va_end(args);						\
8624 }								\
8625 EXPORT_SYMBOL(func);
8626 
8627 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8628 define_netdev_printk_level(netdev_alert, KERN_ALERT);
8629 define_netdev_printk_level(netdev_crit, KERN_CRIT);
8630 define_netdev_printk_level(netdev_err, KERN_ERR);
8631 define_netdev_printk_level(netdev_warn, KERN_WARNING);
8632 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8633 define_netdev_printk_level(netdev_info, KERN_INFO);
8634 
netdev_exit(struct net * net)8635 static void __net_exit netdev_exit(struct net *net)
8636 {
8637 	kfree(net->dev_name_head);
8638 	kfree(net->dev_index_head);
8639 }
8640 
8641 static struct pernet_operations __net_initdata netdev_net_ops = {
8642 	.init = netdev_init,
8643 	.exit = netdev_exit,
8644 };
8645 
default_device_exit(struct net * net)8646 static void __net_exit default_device_exit(struct net *net)
8647 {
8648 	struct net_device *dev, *aux;
8649 	/*
8650 	 * Push all migratable network devices back to the
8651 	 * initial network namespace
8652 	 */
8653 	rtnl_lock();
8654 	for_each_netdev_safe(net, dev, aux) {
8655 		int err;
8656 		char fb_name[IFNAMSIZ];
8657 
8658 		/* Ignore unmoveable devices (i.e. loopback) */
8659 		if (dev->features & NETIF_F_NETNS_LOCAL)
8660 			continue;
8661 
8662 		/* Leave virtual devices for the generic cleanup */
8663 		if (dev->rtnl_link_ops)
8664 			continue;
8665 
8666 		/* Push remaining network devices to init_net */
8667 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8668 		if (__dev_get_by_name(&init_net, fb_name))
8669 			snprintf(fb_name, IFNAMSIZ, "dev%%d");
8670 		err = dev_change_net_namespace(dev, &init_net, fb_name);
8671 		if (err) {
8672 			pr_emerg("%s: failed to move %s to init_net: %d\n",
8673 				 __func__, dev->name, err);
8674 			BUG();
8675 		}
8676 	}
8677 	rtnl_unlock();
8678 }
8679 
rtnl_lock_unregistering(struct list_head * net_list)8680 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8681 {
8682 	/* Return with the rtnl_lock held when there are no network
8683 	 * devices unregistering in any network namespace in net_list.
8684 	 */
8685 	struct net *net;
8686 	bool unregistering;
8687 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
8688 
8689 	add_wait_queue(&netdev_unregistering_wq, &wait);
8690 	for (;;) {
8691 		unregistering = false;
8692 		rtnl_lock();
8693 		list_for_each_entry(net, net_list, exit_list) {
8694 			if (net->dev_unreg_count > 0) {
8695 				unregistering = true;
8696 				break;
8697 			}
8698 		}
8699 		if (!unregistering)
8700 			break;
8701 		__rtnl_unlock();
8702 
8703 		wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8704 	}
8705 	remove_wait_queue(&netdev_unregistering_wq, &wait);
8706 }
8707 
default_device_exit_batch(struct list_head * net_list)8708 static void __net_exit default_device_exit_batch(struct list_head *net_list)
8709 {
8710 	/* At exit all network devices most be removed from a network
8711 	 * namespace.  Do this in the reverse order of registration.
8712 	 * Do this across as many network namespaces as possible to
8713 	 * improve batching efficiency.
8714 	 */
8715 	struct net_device *dev;
8716 	struct net *net;
8717 	LIST_HEAD(dev_kill_list);
8718 
8719 	/* To prevent network device cleanup code from dereferencing
8720 	 * loopback devices or network devices that have been freed
8721 	 * wait here for all pending unregistrations to complete,
8722 	 * before unregistring the loopback device and allowing the
8723 	 * network namespace be freed.
8724 	 *
8725 	 * The netdev todo list containing all network devices
8726 	 * unregistrations that happen in default_device_exit_batch
8727 	 * will run in the rtnl_unlock() at the end of
8728 	 * default_device_exit_batch.
8729 	 */
8730 	rtnl_lock_unregistering(net_list);
8731 	list_for_each_entry(net, net_list, exit_list) {
8732 		for_each_netdev_reverse(net, dev) {
8733 			if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8734 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8735 			else
8736 				unregister_netdevice_queue(dev, &dev_kill_list);
8737 		}
8738 	}
8739 	unregister_netdevice_many(&dev_kill_list);
8740 	rtnl_unlock();
8741 }
8742 
8743 static struct pernet_operations __net_initdata default_device_ops = {
8744 	.exit = default_device_exit,
8745 	.exit_batch = default_device_exit_batch,
8746 };
8747 
8748 /*
8749  *	Initialize the DEV module. At boot time this walks the device list and
8750  *	unhooks any devices that fail to initialise (normally hardware not
8751  *	present) and leaves us with a valid list of present and active devices.
8752  *
8753  */
8754 
8755 /*
8756  *       This is called single threaded during boot, so no need
8757  *       to take the rtnl semaphore.
8758  */
net_dev_init(void)8759 static int __init net_dev_init(void)
8760 {
8761 	int i, rc = -ENOMEM;
8762 
8763 	BUG_ON(!dev_boot_phase);
8764 
8765 	if (dev_proc_init())
8766 		goto out;
8767 
8768 	if (netdev_kobject_init())
8769 		goto out;
8770 
8771 	INIT_LIST_HEAD(&ptype_all);
8772 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
8773 		INIT_LIST_HEAD(&ptype_base[i]);
8774 
8775 	INIT_LIST_HEAD(&offload_base);
8776 
8777 	if (register_pernet_subsys(&netdev_net_ops))
8778 		goto out;
8779 
8780 	/*
8781 	 *	Initialise the packet receive queues.
8782 	 */
8783 
8784 	for_each_possible_cpu(i) {
8785 		struct work_struct *flush = per_cpu_ptr(&flush_works, i);
8786 		struct softnet_data *sd = &per_cpu(softnet_data, i);
8787 
8788 		INIT_WORK(flush, flush_backlog);
8789 
8790 		skb_queue_head_init(&sd->input_pkt_queue);
8791 		skb_queue_head_init(&sd->process_queue);
8792 		INIT_LIST_HEAD(&sd->poll_list);
8793 		sd->output_queue_tailp = &sd->output_queue;
8794 #ifdef CONFIG_RPS
8795 		sd->csd.func = rps_trigger_softirq;
8796 		sd->csd.info = sd;
8797 		sd->cpu = i;
8798 #endif
8799 
8800 		sd->backlog.poll = process_backlog;
8801 		sd->backlog.weight = weight_p;
8802 	}
8803 
8804 	dev_boot_phase = 0;
8805 
8806 	/* The loopback device is special if any other network devices
8807 	 * is present in a network namespace the loopback device must
8808 	 * be present. Since we now dynamically allocate and free the
8809 	 * loopback device ensure this invariant is maintained by
8810 	 * keeping the loopback device as the first device on the
8811 	 * list of network devices.  Ensuring the loopback devices
8812 	 * is the first device that appears and the last network device
8813 	 * that disappears.
8814 	 */
8815 	if (register_pernet_device(&loopback_net_ops))
8816 		goto out;
8817 
8818 	if (register_pernet_device(&default_device_ops))
8819 		goto out;
8820 
8821 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8822 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8823 
8824 	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8825 				       NULL, dev_cpu_dead);
8826 	WARN_ON(rc < 0);
8827 	rc = 0;
8828 out:
8829 	return rc;
8830 }
8831 
8832 subsys_initcall(net_dev_init);
8833