1 /*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75 #include <linux/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/sched/mm.h>
85 #include <linux/mutex.h>
86 #include <linux/string.h>
87 #include <linux/mm.h>
88 #include <linux/socket.h>
89 #include <linux/sockios.h>
90 #include <linux/errno.h>
91 #include <linux/interrupt.h>
92 #include <linux/if_ether.h>
93 #include <linux/netdevice.h>
94 #include <linux/etherdevice.h>
95 #include <linux/ethtool.h>
96 #include <linux/notifier.h>
97 #include <linux/skbuff.h>
98 #include <linux/bpf.h>
99 #include <linux/bpf_trace.h>
100 #include <net/net_namespace.h>
101 #include <net/sock.h>
102 #include <net/busy_poll.h>
103 #include <linux/rtnetlink.h>
104 #include <linux/stat.h>
105 #include <net/dst.h>
106 #include <net/dst_metadata.h>
107 #include <net/pkt_sched.h>
108 #include <net/pkt_cls.h>
109 #include <net/checksum.h>
110 #include <net/xfrm.h>
111 #include <linux/highmem.h>
112 #include <linux/init.h>
113 #include <linux/module.h>
114 #include <linux/netpoll.h>
115 #include <linux/rcupdate.h>
116 #include <linux/delay.h>
117 #include <net/iw_handler.h>
118 #include <asm/current.h>
119 #include <linux/audit.h>
120 #include <linux/dmaengine.h>
121 #include <linux/err.h>
122 #include <linux/ctype.h>
123 #include <linux/if_arp.h>
124 #include <linux/if_vlan.h>
125 #include <linux/ip.h>
126 #include <net/ip.h>
127 #include <net/mpls.h>
128 #include <linux/ipv6.h>
129 #include <linux/in.h>
130 #include <linux/jhash.h>
131 #include <linux/random.h>
132 #include <trace/events/napi.h>
133 #include <trace/events/net.h>
134 #include <trace/events/skb.h>
135 #include <linux/pci.h>
136 #include <linux/inetdevice.h>
137 #include <linux/cpu_rmap.h>
138 #include <linux/static_key.h>
139 #include <linux/hashtable.h>
140 #include <linux/vmalloc.h>
141 #include <linux/if_macvlan.h>
142 #include <linux/errqueue.h>
143 #include <linux/hrtimer.h>
144 #include <linux/netfilter_ingress.h>
145 #include <linux/crash_dump.h>
146 #include <linux/sctp.h>
147 #include <net/udp_tunnel.h>
148
149 #include "net-sysfs.h"
150
151 /* Instead of increasing this, you should create a hash table. */
152 #define MAX_GRO_SKBS 8
153
154 /* This should be increased if a protocol with a bigger head is added. */
155 #define GRO_MAX_HEAD (MAX_HEADER + 128)
156
157 static DEFINE_SPINLOCK(ptype_lock);
158 static DEFINE_SPINLOCK(offload_lock);
159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160 struct list_head ptype_all __read_mostly; /* Taps */
161 static struct list_head offload_base __read_mostly;
162
163 static int netif_rx_internal(struct sk_buff *skb);
164 static int call_netdevice_notifiers_info(unsigned long val,
165 struct net_device *dev,
166 struct netdev_notifier_info *info);
167 static struct napi_struct *napi_by_id(unsigned int napi_id);
168
169 /*
170 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
171 * semaphore.
172 *
173 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
174 *
175 * Writers must hold the rtnl semaphore while they loop through the
176 * dev_base_head list, and hold dev_base_lock for writing when they do the
177 * actual updates. This allows pure readers to access the list even
178 * while a writer is preparing to update it.
179 *
180 * To put it another way, dev_base_lock is held for writing only to
181 * protect against pure readers; the rtnl semaphore provides the
182 * protection against other writers.
183 *
184 * See, for example usages, register_netdevice() and
185 * unregister_netdevice(), which must be called with the rtnl
186 * semaphore held.
187 */
188 DEFINE_RWLOCK(dev_base_lock);
189 EXPORT_SYMBOL(dev_base_lock);
190
191 /* protects napi_hash addition/deletion and napi_gen_id */
192 static DEFINE_SPINLOCK(napi_hash_lock);
193
194 static unsigned int napi_gen_id = NR_CPUS;
195 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
196
197 static seqcount_t devnet_rename_seq;
198
dev_base_seq_inc(struct net * net)199 static inline void dev_base_seq_inc(struct net *net)
200 {
201 while (++net->dev_base_seq == 0)
202 ;
203 }
204
dev_name_hash(struct net * net,const char * name)205 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
206 {
207 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
208
209 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
210 }
211
dev_index_hash(struct net * net,int ifindex)212 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
213 {
214 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
215 }
216
rps_lock(struct softnet_data * sd)217 static inline void rps_lock(struct softnet_data *sd)
218 {
219 #ifdef CONFIG_RPS
220 spin_lock(&sd->input_pkt_queue.lock);
221 #endif
222 }
223
rps_unlock(struct softnet_data * sd)224 static inline void rps_unlock(struct softnet_data *sd)
225 {
226 #ifdef CONFIG_RPS
227 spin_unlock(&sd->input_pkt_queue.lock);
228 #endif
229 }
230
231 /* Device list insertion */
list_netdevice(struct net_device * dev)232 static void list_netdevice(struct net_device *dev)
233 {
234 struct net *net = dev_net(dev);
235
236 ASSERT_RTNL();
237
238 write_lock_bh(&dev_base_lock);
239 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
240 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
241 hlist_add_head_rcu(&dev->index_hlist,
242 dev_index_hash(net, dev->ifindex));
243 write_unlock_bh(&dev_base_lock);
244
245 dev_base_seq_inc(net);
246 }
247
248 /* Device list removal
249 * caller must respect a RCU grace period before freeing/reusing dev
250 */
unlist_netdevice(struct net_device * dev)251 static void unlist_netdevice(struct net_device *dev)
252 {
253 ASSERT_RTNL();
254
255 /* Unlink dev from the device chain */
256 write_lock_bh(&dev_base_lock);
257 list_del_rcu(&dev->dev_list);
258 hlist_del_rcu(&dev->name_hlist);
259 hlist_del_rcu(&dev->index_hlist);
260 write_unlock_bh(&dev_base_lock);
261
262 dev_base_seq_inc(dev_net(dev));
263 }
264
265 /*
266 * Our notifier list
267 */
268
269 static RAW_NOTIFIER_HEAD(netdev_chain);
270
271 /*
272 * Device drivers call our routines to queue packets here. We empty the
273 * queue in the local softnet handler.
274 */
275
276 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
277 EXPORT_PER_CPU_SYMBOL(softnet_data);
278
279 #ifdef CONFIG_LOCKDEP
280 /*
281 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
282 * according to dev->type
283 */
284 static const unsigned short netdev_lock_type[] = {
285 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
286 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
287 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
288 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
289 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
290 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
291 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
292 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
293 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
294 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
295 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
296 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
297 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
298 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
299 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
300
301 static const char *const netdev_lock_name[] = {
302 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
303 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
304 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
305 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
306 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
307 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
308 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
309 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
310 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
311 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
312 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
313 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
314 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
315 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
316 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
317
318 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
319 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
320
netdev_lock_pos(unsigned short dev_type)321 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
322 {
323 int i;
324
325 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
326 if (netdev_lock_type[i] == dev_type)
327 return i;
328 /* the last key is used by default */
329 return ARRAY_SIZE(netdev_lock_type) - 1;
330 }
331
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)332 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
333 unsigned short dev_type)
334 {
335 int i;
336
337 i = netdev_lock_pos(dev_type);
338 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
339 netdev_lock_name[i]);
340 }
341
netdev_set_addr_lockdep_class(struct net_device * dev)342 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
343 {
344 int i;
345
346 i = netdev_lock_pos(dev->type);
347 lockdep_set_class_and_name(&dev->addr_list_lock,
348 &netdev_addr_lock_key[i],
349 netdev_lock_name[i]);
350 }
351 #else
netdev_set_xmit_lockdep_class(spinlock_t * lock,unsigned short dev_type)352 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
353 unsigned short dev_type)
354 {
355 }
netdev_set_addr_lockdep_class(struct net_device * dev)356 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
357 {
358 }
359 #endif
360
361 /*******************************************************************************
362 *
363 * Protocol management and registration routines
364 *
365 *******************************************************************************/
366
367
368 /*
369 * Add a protocol ID to the list. Now that the input handler is
370 * smarter we can dispense with all the messy stuff that used to be
371 * here.
372 *
373 * BEWARE!!! Protocol handlers, mangling input packets,
374 * MUST BE last in hash buckets and checking protocol handlers
375 * MUST start from promiscuous ptype_all chain in net_bh.
376 * It is true now, do not change it.
377 * Explanation follows: if protocol handler, mangling packet, will
378 * be the first on list, it is not able to sense, that packet
379 * is cloned and should be copied-on-write, so that it will
380 * change it and subsequent readers will get broken packet.
381 * --ANK (980803)
382 */
383
ptype_head(const struct packet_type * pt)384 static inline struct list_head *ptype_head(const struct packet_type *pt)
385 {
386 if (pt->type == htons(ETH_P_ALL))
387 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
388 else
389 return pt->dev ? &pt->dev->ptype_specific :
390 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
391 }
392
393 /**
394 * dev_add_pack - add packet handler
395 * @pt: packet type declaration
396 *
397 * Add a protocol handler to the networking stack. The passed &packet_type
398 * is linked into kernel lists and may not be freed until it has been
399 * removed from the kernel lists.
400 *
401 * This call does not sleep therefore it can not
402 * guarantee all CPU's that are in middle of receiving packets
403 * will see the new packet type (until the next received packet).
404 */
405
dev_add_pack(struct packet_type * pt)406 void dev_add_pack(struct packet_type *pt)
407 {
408 struct list_head *head = ptype_head(pt);
409
410 spin_lock(&ptype_lock);
411 list_add_rcu(&pt->list, head);
412 spin_unlock(&ptype_lock);
413 }
414 EXPORT_SYMBOL(dev_add_pack);
415
416 /**
417 * __dev_remove_pack - remove packet handler
418 * @pt: packet type declaration
419 *
420 * Remove a protocol handler that was previously added to the kernel
421 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
422 * from the kernel lists and can be freed or reused once this function
423 * returns.
424 *
425 * The packet type might still be in use by receivers
426 * and must not be freed until after all the CPU's have gone
427 * through a quiescent state.
428 */
__dev_remove_pack(struct packet_type * pt)429 void __dev_remove_pack(struct packet_type *pt)
430 {
431 struct list_head *head = ptype_head(pt);
432 struct packet_type *pt1;
433
434 spin_lock(&ptype_lock);
435
436 list_for_each_entry(pt1, head, list) {
437 if (pt == pt1) {
438 list_del_rcu(&pt->list);
439 goto out;
440 }
441 }
442
443 pr_warn("dev_remove_pack: %p not found\n", pt);
444 out:
445 spin_unlock(&ptype_lock);
446 }
447 EXPORT_SYMBOL(__dev_remove_pack);
448
449 /**
450 * dev_remove_pack - remove packet handler
451 * @pt: packet type declaration
452 *
453 * Remove a protocol handler that was previously added to the kernel
454 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
455 * from the kernel lists and can be freed or reused once this function
456 * returns.
457 *
458 * This call sleeps to guarantee that no CPU is looking at the packet
459 * type after return.
460 */
dev_remove_pack(struct packet_type * pt)461 void dev_remove_pack(struct packet_type *pt)
462 {
463 __dev_remove_pack(pt);
464
465 synchronize_net();
466 }
467 EXPORT_SYMBOL(dev_remove_pack);
468
469
470 /**
471 * dev_add_offload - register offload handlers
472 * @po: protocol offload declaration
473 *
474 * Add protocol offload handlers to the networking stack. The passed
475 * &proto_offload is linked into kernel lists and may not be freed until
476 * it has been removed from the kernel lists.
477 *
478 * This call does not sleep therefore it can not
479 * guarantee all CPU's that are in middle of receiving packets
480 * will see the new offload handlers (until the next received packet).
481 */
dev_add_offload(struct packet_offload * po)482 void dev_add_offload(struct packet_offload *po)
483 {
484 struct packet_offload *elem;
485
486 spin_lock(&offload_lock);
487 list_for_each_entry(elem, &offload_base, list) {
488 if (po->priority < elem->priority)
489 break;
490 }
491 list_add_rcu(&po->list, elem->list.prev);
492 spin_unlock(&offload_lock);
493 }
494 EXPORT_SYMBOL(dev_add_offload);
495
496 /**
497 * __dev_remove_offload - remove offload handler
498 * @po: packet offload declaration
499 *
500 * Remove a protocol offload handler that was previously added to the
501 * kernel offload handlers by dev_add_offload(). The passed &offload_type
502 * is removed from the kernel lists and can be freed or reused once this
503 * function returns.
504 *
505 * The packet type might still be in use by receivers
506 * and must not be freed until after all the CPU's have gone
507 * through a quiescent state.
508 */
__dev_remove_offload(struct packet_offload * po)509 static void __dev_remove_offload(struct packet_offload *po)
510 {
511 struct list_head *head = &offload_base;
512 struct packet_offload *po1;
513
514 spin_lock(&offload_lock);
515
516 list_for_each_entry(po1, head, list) {
517 if (po == po1) {
518 list_del_rcu(&po->list);
519 goto out;
520 }
521 }
522
523 pr_warn("dev_remove_offload: %p not found\n", po);
524 out:
525 spin_unlock(&offload_lock);
526 }
527
528 /**
529 * dev_remove_offload - remove packet offload handler
530 * @po: packet offload declaration
531 *
532 * Remove a packet offload handler that was previously added to the kernel
533 * offload handlers by dev_add_offload(). The passed &offload_type is
534 * removed from the kernel lists and can be freed or reused once this
535 * function returns.
536 *
537 * This call sleeps to guarantee that no CPU is looking at the packet
538 * type after return.
539 */
dev_remove_offload(struct packet_offload * po)540 void dev_remove_offload(struct packet_offload *po)
541 {
542 __dev_remove_offload(po);
543
544 synchronize_net();
545 }
546 EXPORT_SYMBOL(dev_remove_offload);
547
548 /******************************************************************************
549 *
550 * Device Boot-time Settings Routines
551 *
552 ******************************************************************************/
553
554 /* Boot time configuration table */
555 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
556
557 /**
558 * netdev_boot_setup_add - add new setup entry
559 * @name: name of the device
560 * @map: configured settings for the device
561 *
562 * Adds new setup entry to the dev_boot_setup list. The function
563 * returns 0 on error and 1 on success. This is a generic routine to
564 * all netdevices.
565 */
netdev_boot_setup_add(char * name,struct ifmap * map)566 static int netdev_boot_setup_add(char *name, struct ifmap *map)
567 {
568 struct netdev_boot_setup *s;
569 int i;
570
571 s = dev_boot_setup;
572 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
573 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
574 memset(s[i].name, 0, sizeof(s[i].name));
575 strlcpy(s[i].name, name, IFNAMSIZ);
576 memcpy(&s[i].map, map, sizeof(s[i].map));
577 break;
578 }
579 }
580
581 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
582 }
583
584 /**
585 * netdev_boot_setup_check - check boot time settings
586 * @dev: the netdevice
587 *
588 * Check boot time settings for the device.
589 * The found settings are set for the device to be used
590 * later in the device probing.
591 * Returns 0 if no settings found, 1 if they are.
592 */
netdev_boot_setup_check(struct net_device * dev)593 int netdev_boot_setup_check(struct net_device *dev)
594 {
595 struct netdev_boot_setup *s = dev_boot_setup;
596 int i;
597
598 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
599 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
600 !strcmp(dev->name, s[i].name)) {
601 dev->irq = s[i].map.irq;
602 dev->base_addr = s[i].map.base_addr;
603 dev->mem_start = s[i].map.mem_start;
604 dev->mem_end = s[i].map.mem_end;
605 return 1;
606 }
607 }
608 return 0;
609 }
610 EXPORT_SYMBOL(netdev_boot_setup_check);
611
612
613 /**
614 * netdev_boot_base - get address from boot time settings
615 * @prefix: prefix for network device
616 * @unit: id for network device
617 *
618 * Check boot time settings for the base address of device.
619 * The found settings are set for the device to be used
620 * later in the device probing.
621 * Returns 0 if no settings found.
622 */
netdev_boot_base(const char * prefix,int unit)623 unsigned long netdev_boot_base(const char *prefix, int unit)
624 {
625 const struct netdev_boot_setup *s = dev_boot_setup;
626 char name[IFNAMSIZ];
627 int i;
628
629 sprintf(name, "%s%d", prefix, unit);
630
631 /*
632 * If device already registered then return base of 1
633 * to indicate not to probe for this interface
634 */
635 if (__dev_get_by_name(&init_net, name))
636 return 1;
637
638 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
639 if (!strcmp(name, s[i].name))
640 return s[i].map.base_addr;
641 return 0;
642 }
643
644 /*
645 * Saves at boot time configured settings for any netdevice.
646 */
netdev_boot_setup(char * str)647 int __init netdev_boot_setup(char *str)
648 {
649 int ints[5];
650 struct ifmap map;
651
652 str = get_options(str, ARRAY_SIZE(ints), ints);
653 if (!str || !*str)
654 return 0;
655
656 /* Save settings */
657 memset(&map, 0, sizeof(map));
658 if (ints[0] > 0)
659 map.irq = ints[1];
660 if (ints[0] > 1)
661 map.base_addr = ints[2];
662 if (ints[0] > 2)
663 map.mem_start = ints[3];
664 if (ints[0] > 3)
665 map.mem_end = ints[4];
666
667 /* Add new entry to the list */
668 return netdev_boot_setup_add(str, &map);
669 }
670
671 __setup("netdev=", netdev_boot_setup);
672
673 /*******************************************************************************
674 *
675 * Device Interface Subroutines
676 *
677 *******************************************************************************/
678
679 /**
680 * dev_get_iflink - get 'iflink' value of a interface
681 * @dev: targeted interface
682 *
683 * Indicates the ifindex the interface is linked to.
684 * Physical interfaces have the same 'ifindex' and 'iflink' values.
685 */
686
dev_get_iflink(const struct net_device * dev)687 int dev_get_iflink(const struct net_device *dev)
688 {
689 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
690 return dev->netdev_ops->ndo_get_iflink(dev);
691
692 return dev->ifindex;
693 }
694 EXPORT_SYMBOL(dev_get_iflink);
695
696 /**
697 * dev_fill_metadata_dst - Retrieve tunnel egress information.
698 * @dev: targeted interface
699 * @skb: The packet.
700 *
701 * For better visibility of tunnel traffic OVS needs to retrieve
702 * egress tunnel information for a packet. Following API allows
703 * user to get this info.
704 */
dev_fill_metadata_dst(struct net_device * dev,struct sk_buff * skb)705 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
706 {
707 struct ip_tunnel_info *info;
708
709 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
710 return -EINVAL;
711
712 info = skb_tunnel_info_unclone(skb);
713 if (!info)
714 return -ENOMEM;
715 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
716 return -EINVAL;
717
718 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
719 }
720 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
721
722 /**
723 * __dev_get_by_name - find a device by its name
724 * @net: the applicable net namespace
725 * @name: name to find
726 *
727 * Find an interface by name. Must be called under RTNL semaphore
728 * or @dev_base_lock. If the name is found a pointer to the device
729 * is returned. If the name is not found then %NULL is returned. The
730 * reference counters are not incremented so the caller must be
731 * careful with locks.
732 */
733
__dev_get_by_name(struct net * net,const char * name)734 struct net_device *__dev_get_by_name(struct net *net, const char *name)
735 {
736 struct net_device *dev;
737 struct hlist_head *head = dev_name_hash(net, name);
738
739 hlist_for_each_entry(dev, head, name_hlist)
740 if (!strncmp(dev->name, name, IFNAMSIZ))
741 return dev;
742
743 return NULL;
744 }
745 EXPORT_SYMBOL(__dev_get_by_name);
746
747 /**
748 * dev_get_by_name_rcu - find a device by its name
749 * @net: the applicable net namespace
750 * @name: name to find
751 *
752 * Find an interface by name.
753 * If the name is found a pointer to the device is returned.
754 * If the name is not found then %NULL is returned.
755 * The reference counters are not incremented so the caller must be
756 * careful with locks. The caller must hold RCU lock.
757 */
758
dev_get_by_name_rcu(struct net * net,const char * name)759 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
760 {
761 struct net_device *dev;
762 struct hlist_head *head = dev_name_hash(net, name);
763
764 hlist_for_each_entry_rcu(dev, head, name_hlist)
765 if (!strncmp(dev->name, name, IFNAMSIZ))
766 return dev;
767
768 return NULL;
769 }
770 EXPORT_SYMBOL(dev_get_by_name_rcu);
771
772 /**
773 * dev_get_by_name - find a device by its name
774 * @net: the applicable net namespace
775 * @name: name to find
776 *
777 * Find an interface by name. This can be called from any
778 * context and does its own locking. The returned handle has
779 * the usage count incremented and the caller must use dev_put() to
780 * release it when it is no longer needed. %NULL is returned if no
781 * matching device is found.
782 */
783
dev_get_by_name(struct net * net,const char * name)784 struct net_device *dev_get_by_name(struct net *net, const char *name)
785 {
786 struct net_device *dev;
787
788 rcu_read_lock();
789 dev = dev_get_by_name_rcu(net, name);
790 if (dev)
791 dev_hold(dev);
792 rcu_read_unlock();
793 return dev;
794 }
795 EXPORT_SYMBOL(dev_get_by_name);
796
797 /**
798 * __dev_get_by_index - find a device by its ifindex
799 * @net: the applicable net namespace
800 * @ifindex: index of device
801 *
802 * Search for an interface by index. Returns %NULL if the device
803 * is not found or a pointer to the device. The device has not
804 * had its reference counter increased so the caller must be careful
805 * about locking. The caller must hold either the RTNL semaphore
806 * or @dev_base_lock.
807 */
808
__dev_get_by_index(struct net * net,int ifindex)809 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
810 {
811 struct net_device *dev;
812 struct hlist_head *head = dev_index_hash(net, ifindex);
813
814 hlist_for_each_entry(dev, head, index_hlist)
815 if (dev->ifindex == ifindex)
816 return dev;
817
818 return NULL;
819 }
820 EXPORT_SYMBOL(__dev_get_by_index);
821
822 /**
823 * dev_get_by_index_rcu - find a device by its ifindex
824 * @net: the applicable net namespace
825 * @ifindex: index of device
826 *
827 * Search for an interface by index. Returns %NULL if the device
828 * is not found or a pointer to the device. The device has not
829 * had its reference counter increased so the caller must be careful
830 * about locking. The caller must hold RCU lock.
831 */
832
dev_get_by_index_rcu(struct net * net,int ifindex)833 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
834 {
835 struct net_device *dev;
836 struct hlist_head *head = dev_index_hash(net, ifindex);
837
838 hlist_for_each_entry_rcu(dev, head, index_hlist)
839 if (dev->ifindex == ifindex)
840 return dev;
841
842 return NULL;
843 }
844 EXPORT_SYMBOL(dev_get_by_index_rcu);
845
846
847 /**
848 * dev_get_by_index - find a device by its ifindex
849 * @net: the applicable net namespace
850 * @ifindex: index of device
851 *
852 * Search for an interface by index. Returns NULL if the device
853 * is not found or a pointer to the device. The device returned has
854 * had a reference added and the pointer is safe until the user calls
855 * dev_put to indicate they have finished with it.
856 */
857
dev_get_by_index(struct net * net,int ifindex)858 struct net_device *dev_get_by_index(struct net *net, int ifindex)
859 {
860 struct net_device *dev;
861
862 rcu_read_lock();
863 dev = dev_get_by_index_rcu(net, ifindex);
864 if (dev)
865 dev_hold(dev);
866 rcu_read_unlock();
867 return dev;
868 }
869 EXPORT_SYMBOL(dev_get_by_index);
870
871 /**
872 * dev_get_by_napi_id - find a device by napi_id
873 * @napi_id: ID of the NAPI struct
874 *
875 * Search for an interface by NAPI ID. Returns %NULL if the device
876 * is not found or a pointer to the device. The device has not had
877 * its reference counter increased so the caller must be careful
878 * about locking. The caller must hold RCU lock.
879 */
880
dev_get_by_napi_id(unsigned int napi_id)881 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
882 {
883 struct napi_struct *napi;
884
885 WARN_ON_ONCE(!rcu_read_lock_held());
886
887 if (napi_id < MIN_NAPI_ID)
888 return NULL;
889
890 napi = napi_by_id(napi_id);
891
892 return napi ? napi->dev : NULL;
893 }
894 EXPORT_SYMBOL(dev_get_by_napi_id);
895
896 /**
897 * netdev_get_name - get a netdevice name, knowing its ifindex.
898 * @net: network namespace
899 * @name: a pointer to the buffer where the name will be stored.
900 * @ifindex: the ifindex of the interface to get the name from.
901 *
902 * The use of raw_seqcount_begin() and cond_resched() before
903 * retrying is required as we want to give the writers a chance
904 * to complete when CONFIG_PREEMPT is not set.
905 */
netdev_get_name(struct net * net,char * name,int ifindex)906 int netdev_get_name(struct net *net, char *name, int ifindex)
907 {
908 struct net_device *dev;
909 unsigned int seq;
910
911 retry:
912 seq = raw_seqcount_begin(&devnet_rename_seq);
913 rcu_read_lock();
914 dev = dev_get_by_index_rcu(net, ifindex);
915 if (!dev) {
916 rcu_read_unlock();
917 return -ENODEV;
918 }
919
920 strcpy(name, dev->name);
921 rcu_read_unlock();
922 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
923 cond_resched();
924 goto retry;
925 }
926
927 return 0;
928 }
929
930 /**
931 * dev_getbyhwaddr_rcu - find a device by its hardware address
932 * @net: the applicable net namespace
933 * @type: media type of device
934 * @ha: hardware address
935 *
936 * Search for an interface by MAC address. Returns NULL if the device
937 * is not found or a pointer to the device.
938 * The caller must hold RCU or RTNL.
939 * The returned device has not had its ref count increased
940 * and the caller must therefore be careful about locking
941 *
942 */
943
dev_getbyhwaddr_rcu(struct net * net,unsigned short type,const char * ha)944 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
945 const char *ha)
946 {
947 struct net_device *dev;
948
949 for_each_netdev_rcu(net, dev)
950 if (dev->type == type &&
951 !memcmp(dev->dev_addr, ha, dev->addr_len))
952 return dev;
953
954 return NULL;
955 }
956 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
957
__dev_getfirstbyhwtype(struct net * net,unsigned short type)958 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
959 {
960 struct net_device *dev;
961
962 ASSERT_RTNL();
963 for_each_netdev(net, dev)
964 if (dev->type == type)
965 return dev;
966
967 return NULL;
968 }
969 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
970
dev_getfirstbyhwtype(struct net * net,unsigned short type)971 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
972 {
973 struct net_device *dev, *ret = NULL;
974
975 rcu_read_lock();
976 for_each_netdev_rcu(net, dev)
977 if (dev->type == type) {
978 dev_hold(dev);
979 ret = dev;
980 break;
981 }
982 rcu_read_unlock();
983 return ret;
984 }
985 EXPORT_SYMBOL(dev_getfirstbyhwtype);
986
987 /**
988 * __dev_get_by_flags - find any device with given flags
989 * @net: the applicable net namespace
990 * @if_flags: IFF_* values
991 * @mask: bitmask of bits in if_flags to check
992 *
993 * Search for any interface with the given flags. Returns NULL if a device
994 * is not found or a pointer to the device. Must be called inside
995 * rtnl_lock(), and result refcount is unchanged.
996 */
997
__dev_get_by_flags(struct net * net,unsigned short if_flags,unsigned short mask)998 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
999 unsigned short mask)
1000 {
1001 struct net_device *dev, *ret;
1002
1003 ASSERT_RTNL();
1004
1005 ret = NULL;
1006 for_each_netdev(net, dev) {
1007 if (((dev->flags ^ if_flags) & mask) == 0) {
1008 ret = dev;
1009 break;
1010 }
1011 }
1012 return ret;
1013 }
1014 EXPORT_SYMBOL(__dev_get_by_flags);
1015
1016 /**
1017 * dev_valid_name - check if name is okay for network device
1018 * @name: name string
1019 *
1020 * Network device names need to be valid file names to
1021 * to allow sysfs to work. We also disallow any kind of
1022 * whitespace.
1023 */
dev_valid_name(const char * name)1024 bool dev_valid_name(const char *name)
1025 {
1026 if (*name == '\0')
1027 return false;
1028 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1029 return false;
1030 if (!strcmp(name, ".") || !strcmp(name, ".."))
1031 return false;
1032
1033 while (*name) {
1034 if (*name == '/' || *name == ':' || isspace(*name))
1035 return false;
1036 name++;
1037 }
1038 return true;
1039 }
1040 EXPORT_SYMBOL(dev_valid_name);
1041
1042 /**
1043 * __dev_alloc_name - allocate a name for a device
1044 * @net: network namespace to allocate the device name in
1045 * @name: name format string
1046 * @buf: scratch buffer and result name string
1047 *
1048 * Passed a format string - eg "lt%d" it will try and find a suitable
1049 * id. It scans list of devices to build up a free map, then chooses
1050 * the first empty slot. The caller must hold the dev_base or rtnl lock
1051 * while allocating the name and adding the device in order to avoid
1052 * duplicates.
1053 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1054 * Returns the number of the unit assigned or a negative errno code.
1055 */
1056
__dev_alloc_name(struct net * net,const char * name,char * buf)1057 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1058 {
1059 int i = 0;
1060 const char *p;
1061 const int max_netdevices = 8*PAGE_SIZE;
1062 unsigned long *inuse;
1063 struct net_device *d;
1064
1065 p = strnchr(name, IFNAMSIZ-1, '%');
1066 if (p) {
1067 /*
1068 * Verify the string as this thing may have come from
1069 * the user. There must be either one "%d" and no other "%"
1070 * characters.
1071 */
1072 if (p[1] != 'd' || strchr(p + 2, '%'))
1073 return -EINVAL;
1074
1075 /* Use one page as a bit array of possible slots */
1076 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1077 if (!inuse)
1078 return -ENOMEM;
1079
1080 for_each_netdev(net, d) {
1081 if (!sscanf(d->name, name, &i))
1082 continue;
1083 if (i < 0 || i >= max_netdevices)
1084 continue;
1085
1086 /* avoid cases where sscanf is not exact inverse of printf */
1087 snprintf(buf, IFNAMSIZ, name, i);
1088 if (!strncmp(buf, d->name, IFNAMSIZ))
1089 set_bit(i, inuse);
1090 }
1091
1092 i = find_first_zero_bit(inuse, max_netdevices);
1093 free_page((unsigned long) inuse);
1094 }
1095
1096 if (buf != name)
1097 snprintf(buf, IFNAMSIZ, name, i);
1098 if (!__dev_get_by_name(net, buf))
1099 return i;
1100
1101 /* It is possible to run out of possible slots
1102 * when the name is long and there isn't enough space left
1103 * for the digits, or if all bits are used.
1104 */
1105 return -ENFILE;
1106 }
1107
1108 /**
1109 * dev_alloc_name - allocate a name for a device
1110 * @dev: device
1111 * @name: name format string
1112 *
1113 * Passed a format string - eg "lt%d" it will try and find a suitable
1114 * id. It scans list of devices to build up a free map, then chooses
1115 * the first empty slot. The caller must hold the dev_base or rtnl lock
1116 * while allocating the name and adding the device in order to avoid
1117 * duplicates.
1118 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1119 * Returns the number of the unit assigned or a negative errno code.
1120 */
1121
dev_alloc_name(struct net_device * dev,const char * name)1122 int dev_alloc_name(struct net_device *dev, const char *name)
1123 {
1124 char buf[IFNAMSIZ];
1125 struct net *net;
1126 int ret;
1127
1128 BUG_ON(!dev_net(dev));
1129 net = dev_net(dev);
1130 ret = __dev_alloc_name(net, name, buf);
1131 if (ret >= 0)
1132 strlcpy(dev->name, buf, IFNAMSIZ);
1133 return ret;
1134 }
1135 EXPORT_SYMBOL(dev_alloc_name);
1136
dev_alloc_name_ns(struct net * net,struct net_device * dev,const char * name)1137 static int dev_alloc_name_ns(struct net *net,
1138 struct net_device *dev,
1139 const char *name)
1140 {
1141 char buf[IFNAMSIZ];
1142 int ret;
1143
1144 ret = __dev_alloc_name(net, name, buf);
1145 if (ret >= 0)
1146 strlcpy(dev->name, buf, IFNAMSIZ);
1147 return ret;
1148 }
1149
dev_get_valid_name(struct net * net,struct net_device * dev,const char * name)1150 int dev_get_valid_name(struct net *net, struct net_device *dev,
1151 const char *name)
1152 {
1153 BUG_ON(!net);
1154
1155 if (!dev_valid_name(name))
1156 return -EINVAL;
1157
1158 if (strchr(name, '%'))
1159 return dev_alloc_name_ns(net, dev, name);
1160 else if (__dev_get_by_name(net, name))
1161 return -EEXIST;
1162 else if (dev->name != name)
1163 strlcpy(dev->name, name, IFNAMSIZ);
1164
1165 return 0;
1166 }
1167 EXPORT_SYMBOL(dev_get_valid_name);
1168
1169 /**
1170 * dev_change_name - change name of a device
1171 * @dev: device
1172 * @newname: name (or format string) must be at least IFNAMSIZ
1173 *
1174 * Change name of a device, can pass format strings "eth%d".
1175 * for wildcarding.
1176 */
dev_change_name(struct net_device * dev,const char * newname)1177 int dev_change_name(struct net_device *dev, const char *newname)
1178 {
1179 unsigned char old_assign_type;
1180 char oldname[IFNAMSIZ];
1181 int err = 0;
1182 int ret;
1183 struct net *net;
1184
1185 ASSERT_RTNL();
1186 BUG_ON(!dev_net(dev));
1187
1188 net = dev_net(dev);
1189 if (dev->flags & IFF_UP)
1190 return -EBUSY;
1191
1192 write_seqcount_begin(&devnet_rename_seq);
1193
1194 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1195 write_seqcount_end(&devnet_rename_seq);
1196 return 0;
1197 }
1198
1199 memcpy(oldname, dev->name, IFNAMSIZ);
1200
1201 err = dev_get_valid_name(net, dev, newname);
1202 if (err < 0) {
1203 write_seqcount_end(&devnet_rename_seq);
1204 return err;
1205 }
1206
1207 if (oldname[0] && !strchr(oldname, '%'))
1208 netdev_info(dev, "renamed from %s\n", oldname);
1209
1210 old_assign_type = dev->name_assign_type;
1211 dev->name_assign_type = NET_NAME_RENAMED;
1212
1213 rollback:
1214 ret = device_rename(&dev->dev, dev->name);
1215 if (ret) {
1216 memcpy(dev->name, oldname, IFNAMSIZ);
1217 dev->name_assign_type = old_assign_type;
1218 write_seqcount_end(&devnet_rename_seq);
1219 return ret;
1220 }
1221
1222 write_seqcount_end(&devnet_rename_seq);
1223
1224 netdev_adjacent_rename_links(dev, oldname);
1225
1226 write_lock_bh(&dev_base_lock);
1227 hlist_del_rcu(&dev->name_hlist);
1228 write_unlock_bh(&dev_base_lock);
1229
1230 synchronize_rcu();
1231
1232 write_lock_bh(&dev_base_lock);
1233 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1234 write_unlock_bh(&dev_base_lock);
1235
1236 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1237 ret = notifier_to_errno(ret);
1238
1239 if (ret) {
1240 /* err >= 0 after dev_alloc_name() or stores the first errno */
1241 if (err >= 0) {
1242 err = ret;
1243 write_seqcount_begin(&devnet_rename_seq);
1244 memcpy(dev->name, oldname, IFNAMSIZ);
1245 memcpy(oldname, newname, IFNAMSIZ);
1246 dev->name_assign_type = old_assign_type;
1247 old_assign_type = NET_NAME_RENAMED;
1248 goto rollback;
1249 } else {
1250 pr_err("%s: name change rollback failed: %d\n",
1251 dev->name, ret);
1252 }
1253 }
1254
1255 return err;
1256 }
1257
1258 /**
1259 * dev_set_alias - change ifalias of a device
1260 * @dev: device
1261 * @alias: name up to IFALIASZ
1262 * @len: limit of bytes to copy from info
1263 *
1264 * Set ifalias for a device,
1265 */
dev_set_alias(struct net_device * dev,const char * alias,size_t len)1266 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1267 {
1268 char *new_ifalias;
1269
1270 ASSERT_RTNL();
1271
1272 if (len >= IFALIASZ)
1273 return -EINVAL;
1274
1275 if (!len) {
1276 kfree(dev->ifalias);
1277 dev->ifalias = NULL;
1278 return 0;
1279 }
1280
1281 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1282 if (!new_ifalias)
1283 return -ENOMEM;
1284 dev->ifalias = new_ifalias;
1285 memcpy(dev->ifalias, alias, len);
1286 dev->ifalias[len] = 0;
1287
1288 return len;
1289 }
1290
1291
1292 /**
1293 * netdev_features_change - device changes features
1294 * @dev: device to cause notification
1295 *
1296 * Called to indicate a device has changed features.
1297 */
netdev_features_change(struct net_device * dev)1298 void netdev_features_change(struct net_device *dev)
1299 {
1300 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1301 }
1302 EXPORT_SYMBOL(netdev_features_change);
1303
1304 /**
1305 * netdev_state_change - device changes state
1306 * @dev: device to cause notification
1307 *
1308 * Called to indicate a device has changed state. This function calls
1309 * the notifier chains for netdev_chain and sends a NEWLINK message
1310 * to the routing socket.
1311 */
netdev_state_change(struct net_device * dev)1312 void netdev_state_change(struct net_device *dev)
1313 {
1314 if (dev->flags & IFF_UP) {
1315 struct netdev_notifier_change_info change_info;
1316
1317 change_info.flags_changed = 0;
1318 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1319 &change_info.info);
1320 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1321 }
1322 }
1323 EXPORT_SYMBOL(netdev_state_change);
1324
1325 /**
1326 * netdev_notify_peers - notify network peers about existence of @dev
1327 * @dev: network device
1328 *
1329 * Generate traffic such that interested network peers are aware of
1330 * @dev, such as by generating a gratuitous ARP. This may be used when
1331 * a device wants to inform the rest of the network about some sort of
1332 * reconfiguration such as a failover event or virtual machine
1333 * migration.
1334 */
netdev_notify_peers(struct net_device * dev)1335 void netdev_notify_peers(struct net_device *dev)
1336 {
1337 rtnl_lock();
1338 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1339 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1340 rtnl_unlock();
1341 }
1342 EXPORT_SYMBOL(netdev_notify_peers);
1343
__dev_open(struct net_device * dev)1344 static int __dev_open(struct net_device *dev)
1345 {
1346 const struct net_device_ops *ops = dev->netdev_ops;
1347 int ret;
1348
1349 ASSERT_RTNL();
1350
1351 if (!netif_device_present(dev))
1352 return -ENODEV;
1353
1354 /* Block netpoll from trying to do any rx path servicing.
1355 * If we don't do this there is a chance ndo_poll_controller
1356 * or ndo_poll may be running while we open the device
1357 */
1358 netpoll_poll_disable(dev);
1359
1360 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1361 ret = notifier_to_errno(ret);
1362 if (ret)
1363 return ret;
1364
1365 set_bit(__LINK_STATE_START, &dev->state);
1366
1367 if (ops->ndo_validate_addr)
1368 ret = ops->ndo_validate_addr(dev);
1369
1370 if (!ret && ops->ndo_open)
1371 ret = ops->ndo_open(dev);
1372
1373 netpoll_poll_enable(dev);
1374
1375 if (ret)
1376 clear_bit(__LINK_STATE_START, &dev->state);
1377 else {
1378 dev->flags |= IFF_UP;
1379 dev_set_rx_mode(dev);
1380 dev_activate(dev);
1381 add_device_randomness(dev->dev_addr, dev->addr_len);
1382 }
1383
1384 return ret;
1385 }
1386
1387 /**
1388 * dev_open - prepare an interface for use.
1389 * @dev: device to open
1390 *
1391 * Takes a device from down to up state. The device's private open
1392 * function is invoked and then the multicast lists are loaded. Finally
1393 * the device is moved into the up state and a %NETDEV_UP message is
1394 * sent to the netdev notifier chain.
1395 *
1396 * Calling this function on an active interface is a nop. On a failure
1397 * a negative errno code is returned.
1398 */
dev_open(struct net_device * dev)1399 int dev_open(struct net_device *dev)
1400 {
1401 int ret;
1402
1403 if (dev->flags & IFF_UP)
1404 return 0;
1405
1406 ret = __dev_open(dev);
1407 if (ret < 0)
1408 return ret;
1409
1410 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1411 call_netdevice_notifiers(NETDEV_UP, dev);
1412
1413 return ret;
1414 }
1415 EXPORT_SYMBOL(dev_open);
1416
__dev_close_many(struct list_head * head)1417 static void __dev_close_many(struct list_head *head)
1418 {
1419 struct net_device *dev;
1420
1421 ASSERT_RTNL();
1422 might_sleep();
1423
1424 list_for_each_entry(dev, head, close_list) {
1425 /* Temporarily disable netpoll until the interface is down */
1426 netpoll_poll_disable(dev);
1427
1428 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1429
1430 clear_bit(__LINK_STATE_START, &dev->state);
1431
1432 /* Synchronize to scheduled poll. We cannot touch poll list, it
1433 * can be even on different cpu. So just clear netif_running().
1434 *
1435 * dev->stop() will invoke napi_disable() on all of it's
1436 * napi_struct instances on this device.
1437 */
1438 smp_mb__after_atomic(); /* Commit netif_running(). */
1439 }
1440
1441 dev_deactivate_many(head);
1442
1443 list_for_each_entry(dev, head, close_list) {
1444 const struct net_device_ops *ops = dev->netdev_ops;
1445
1446 /*
1447 * Call the device specific close. This cannot fail.
1448 * Only if device is UP
1449 *
1450 * We allow it to be called even after a DETACH hot-plug
1451 * event.
1452 */
1453 if (ops->ndo_stop)
1454 ops->ndo_stop(dev);
1455
1456 dev->flags &= ~IFF_UP;
1457 netpoll_poll_enable(dev);
1458 }
1459 }
1460
__dev_close(struct net_device * dev)1461 static void __dev_close(struct net_device *dev)
1462 {
1463 LIST_HEAD(single);
1464
1465 list_add(&dev->close_list, &single);
1466 __dev_close_many(&single);
1467 list_del(&single);
1468 }
1469
dev_close_many(struct list_head * head,bool unlink)1470 void dev_close_many(struct list_head *head, bool unlink)
1471 {
1472 struct net_device *dev, *tmp;
1473
1474 /* Remove the devices that don't need to be closed */
1475 list_for_each_entry_safe(dev, tmp, head, close_list)
1476 if (!(dev->flags & IFF_UP))
1477 list_del_init(&dev->close_list);
1478
1479 __dev_close_many(head);
1480
1481 list_for_each_entry_safe(dev, tmp, head, close_list) {
1482 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1483 call_netdevice_notifiers(NETDEV_DOWN, dev);
1484 if (unlink)
1485 list_del_init(&dev->close_list);
1486 }
1487 }
1488 EXPORT_SYMBOL(dev_close_many);
1489
1490 /**
1491 * dev_close - shutdown an interface.
1492 * @dev: device to shutdown
1493 *
1494 * This function moves an active device into down state. A
1495 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1496 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1497 * chain.
1498 */
dev_close(struct net_device * dev)1499 void dev_close(struct net_device *dev)
1500 {
1501 if (dev->flags & IFF_UP) {
1502 LIST_HEAD(single);
1503
1504 list_add(&dev->close_list, &single);
1505 dev_close_many(&single, true);
1506 list_del(&single);
1507 }
1508 }
1509 EXPORT_SYMBOL(dev_close);
1510
1511
1512 /**
1513 * dev_disable_lro - disable Large Receive Offload on a device
1514 * @dev: device
1515 *
1516 * Disable Large Receive Offload (LRO) on a net device. Must be
1517 * called under RTNL. This is needed if received packets may be
1518 * forwarded to another interface.
1519 */
dev_disable_lro(struct net_device * dev)1520 void dev_disable_lro(struct net_device *dev)
1521 {
1522 struct net_device *lower_dev;
1523 struct list_head *iter;
1524
1525 dev->wanted_features &= ~NETIF_F_LRO;
1526 netdev_update_features(dev);
1527
1528 if (unlikely(dev->features & NETIF_F_LRO))
1529 netdev_WARN(dev, "failed to disable LRO!\n");
1530
1531 netdev_for_each_lower_dev(dev, lower_dev, iter)
1532 dev_disable_lro(lower_dev);
1533 }
1534 EXPORT_SYMBOL(dev_disable_lro);
1535
call_netdevice_notifier(struct notifier_block * nb,unsigned long val,struct net_device * dev)1536 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1537 struct net_device *dev)
1538 {
1539 struct netdev_notifier_info info;
1540
1541 netdev_notifier_info_init(&info, dev);
1542 return nb->notifier_call(nb, val, &info);
1543 }
1544
1545 static int dev_boot_phase = 1;
1546
1547 /**
1548 * register_netdevice_notifier - register a network notifier block
1549 * @nb: notifier
1550 *
1551 * Register a notifier to be called when network device events occur.
1552 * The notifier passed is linked into the kernel structures and must
1553 * not be reused until it has been unregistered. A negative errno code
1554 * is returned on a failure.
1555 *
1556 * When registered all registration and up events are replayed
1557 * to the new notifier to allow device to have a race free
1558 * view of the network device list.
1559 */
1560
register_netdevice_notifier(struct notifier_block * nb)1561 int register_netdevice_notifier(struct notifier_block *nb)
1562 {
1563 struct net_device *dev;
1564 struct net_device *last;
1565 struct net *net;
1566 int err;
1567
1568 rtnl_lock();
1569 err = raw_notifier_chain_register(&netdev_chain, nb);
1570 if (err)
1571 goto unlock;
1572 if (dev_boot_phase)
1573 goto unlock;
1574 for_each_net(net) {
1575 for_each_netdev(net, dev) {
1576 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1577 err = notifier_to_errno(err);
1578 if (err)
1579 goto rollback;
1580
1581 if (!(dev->flags & IFF_UP))
1582 continue;
1583
1584 call_netdevice_notifier(nb, NETDEV_UP, dev);
1585 }
1586 }
1587
1588 unlock:
1589 rtnl_unlock();
1590 return err;
1591
1592 rollback:
1593 last = dev;
1594 for_each_net(net) {
1595 for_each_netdev(net, dev) {
1596 if (dev == last)
1597 goto outroll;
1598
1599 if (dev->flags & IFF_UP) {
1600 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1601 dev);
1602 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1603 }
1604 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1605 }
1606 }
1607
1608 outroll:
1609 raw_notifier_chain_unregister(&netdev_chain, nb);
1610 goto unlock;
1611 }
1612 EXPORT_SYMBOL(register_netdevice_notifier);
1613
1614 /**
1615 * unregister_netdevice_notifier - unregister a network notifier block
1616 * @nb: notifier
1617 *
1618 * Unregister a notifier previously registered by
1619 * register_netdevice_notifier(). The notifier is unlinked into the
1620 * kernel structures and may then be reused. A negative errno code
1621 * is returned on a failure.
1622 *
1623 * After unregistering unregister and down device events are synthesized
1624 * for all devices on the device list to the removed notifier to remove
1625 * the need for special case cleanup code.
1626 */
1627
unregister_netdevice_notifier(struct notifier_block * nb)1628 int unregister_netdevice_notifier(struct notifier_block *nb)
1629 {
1630 struct net_device *dev;
1631 struct net *net;
1632 int err;
1633
1634 rtnl_lock();
1635 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1636 if (err)
1637 goto unlock;
1638
1639 for_each_net(net) {
1640 for_each_netdev(net, dev) {
1641 if (dev->flags & IFF_UP) {
1642 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1643 dev);
1644 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1645 }
1646 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1647 }
1648 }
1649 unlock:
1650 rtnl_unlock();
1651 return err;
1652 }
1653 EXPORT_SYMBOL(unregister_netdevice_notifier);
1654
1655 /**
1656 * call_netdevice_notifiers_info - call all network notifier blocks
1657 * @val: value passed unmodified to notifier function
1658 * @dev: net_device pointer passed unmodified to notifier function
1659 * @info: notifier information data
1660 *
1661 * Call all network notifier blocks. Parameters and return value
1662 * are as for raw_notifier_call_chain().
1663 */
1664
call_netdevice_notifiers_info(unsigned long val,struct net_device * dev,struct netdev_notifier_info * info)1665 static int call_netdevice_notifiers_info(unsigned long val,
1666 struct net_device *dev,
1667 struct netdev_notifier_info *info)
1668 {
1669 ASSERT_RTNL();
1670 netdev_notifier_info_init(info, dev);
1671 return raw_notifier_call_chain(&netdev_chain, val, info);
1672 }
1673
1674 /**
1675 * call_netdevice_notifiers - call all network notifier blocks
1676 * @val: value passed unmodified to notifier function
1677 * @dev: net_device pointer passed unmodified to notifier function
1678 *
1679 * Call all network notifier blocks. Parameters and return value
1680 * are as for raw_notifier_call_chain().
1681 */
1682
call_netdevice_notifiers(unsigned long val,struct net_device * dev)1683 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1684 {
1685 struct netdev_notifier_info info;
1686
1687 return call_netdevice_notifiers_info(val, dev, &info);
1688 }
1689 EXPORT_SYMBOL(call_netdevice_notifiers);
1690
1691 /**
1692 * call_netdevice_notifiers_mtu - call all network notifier blocks
1693 * @val: value passed unmodified to notifier function
1694 * @dev: net_device pointer passed unmodified to notifier function
1695 * @arg: additional u32 argument passed to the notifier function
1696 *
1697 * Call all network notifier blocks. Parameters and return value
1698 * are as for raw_notifier_call_chain().
1699 */
call_netdevice_notifiers_mtu(unsigned long val,struct net_device * dev,u32 arg)1700 static int call_netdevice_notifiers_mtu(unsigned long val,
1701 struct net_device *dev, u32 arg)
1702 {
1703 struct netdev_notifier_info_ext info = {
1704 .info.dev = dev,
1705 .ext.mtu = arg,
1706 };
1707
1708 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
1709
1710 return call_netdevice_notifiers_info(val, dev, &info.info);
1711 }
1712
1713 #ifdef CONFIG_NET_INGRESS
1714 static struct static_key ingress_needed __read_mostly;
1715
net_inc_ingress_queue(void)1716 void net_inc_ingress_queue(void)
1717 {
1718 static_key_slow_inc(&ingress_needed);
1719 }
1720 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1721
net_dec_ingress_queue(void)1722 void net_dec_ingress_queue(void)
1723 {
1724 static_key_slow_dec(&ingress_needed);
1725 }
1726 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1727 #endif
1728
1729 #ifdef CONFIG_NET_EGRESS
1730 static struct static_key egress_needed __read_mostly;
1731
net_inc_egress_queue(void)1732 void net_inc_egress_queue(void)
1733 {
1734 static_key_slow_inc(&egress_needed);
1735 }
1736 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1737
net_dec_egress_queue(void)1738 void net_dec_egress_queue(void)
1739 {
1740 static_key_slow_dec(&egress_needed);
1741 }
1742 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1743 #endif
1744
1745 static struct static_key netstamp_needed __read_mostly;
1746 #ifdef HAVE_JUMP_LABEL
1747 static atomic_t netstamp_needed_deferred;
1748 static atomic_t netstamp_wanted;
netstamp_clear(struct work_struct * work)1749 static void netstamp_clear(struct work_struct *work)
1750 {
1751 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1752 int wanted;
1753
1754 wanted = atomic_add_return(deferred, &netstamp_wanted);
1755 if (wanted > 0)
1756 static_key_enable(&netstamp_needed);
1757 else
1758 static_key_disable(&netstamp_needed);
1759 }
1760 static DECLARE_WORK(netstamp_work, netstamp_clear);
1761 #endif
1762
net_enable_timestamp(void)1763 void net_enable_timestamp(void)
1764 {
1765 #ifdef HAVE_JUMP_LABEL
1766 int wanted;
1767
1768 while (1) {
1769 wanted = atomic_read(&netstamp_wanted);
1770 if (wanted <= 0)
1771 break;
1772 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1773 return;
1774 }
1775 atomic_inc(&netstamp_needed_deferred);
1776 schedule_work(&netstamp_work);
1777 #else
1778 static_key_slow_inc(&netstamp_needed);
1779 #endif
1780 }
1781 EXPORT_SYMBOL(net_enable_timestamp);
1782
net_disable_timestamp(void)1783 void net_disable_timestamp(void)
1784 {
1785 #ifdef HAVE_JUMP_LABEL
1786 int wanted;
1787
1788 while (1) {
1789 wanted = atomic_read(&netstamp_wanted);
1790 if (wanted <= 1)
1791 break;
1792 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1793 return;
1794 }
1795 atomic_dec(&netstamp_needed_deferred);
1796 schedule_work(&netstamp_work);
1797 #else
1798 static_key_slow_dec(&netstamp_needed);
1799 #endif
1800 }
1801 EXPORT_SYMBOL(net_disable_timestamp);
1802
net_timestamp_set(struct sk_buff * skb)1803 static inline void net_timestamp_set(struct sk_buff *skb)
1804 {
1805 skb->tstamp = 0;
1806 if (static_key_false(&netstamp_needed))
1807 __net_timestamp(skb);
1808 }
1809
1810 #define net_timestamp_check(COND, SKB) \
1811 if (static_key_false(&netstamp_needed)) { \
1812 if ((COND) && !(SKB)->tstamp) \
1813 __net_timestamp(SKB); \
1814 } \
1815
is_skb_forwardable(const struct net_device * dev,const struct sk_buff * skb)1816 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1817 {
1818 unsigned int len;
1819
1820 if (!(dev->flags & IFF_UP))
1821 return false;
1822
1823 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1824 if (skb->len <= len)
1825 return true;
1826
1827 /* if TSO is enabled, we don't care about the length as the packet
1828 * could be forwarded without being segmented before
1829 */
1830 if (skb_is_gso(skb))
1831 return true;
1832
1833 return false;
1834 }
1835 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1836
__dev_forward_skb(struct net_device * dev,struct sk_buff * skb)1837 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1838 {
1839 int ret = ____dev_forward_skb(dev, skb);
1840
1841 if (likely(!ret)) {
1842 skb->protocol = eth_type_trans(skb, dev);
1843 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1844 }
1845
1846 return ret;
1847 }
1848 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1849
1850 /**
1851 * dev_forward_skb - loopback an skb to another netif
1852 *
1853 * @dev: destination network device
1854 * @skb: buffer to forward
1855 *
1856 * return values:
1857 * NET_RX_SUCCESS (no congestion)
1858 * NET_RX_DROP (packet was dropped, but freed)
1859 *
1860 * dev_forward_skb can be used for injecting an skb from the
1861 * start_xmit function of one device into the receive queue
1862 * of another device.
1863 *
1864 * The receiving device may be in another namespace, so
1865 * we have to clear all information in the skb that could
1866 * impact namespace isolation.
1867 */
dev_forward_skb(struct net_device * dev,struct sk_buff * skb)1868 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1869 {
1870 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1871 }
1872 EXPORT_SYMBOL_GPL(dev_forward_skb);
1873
deliver_skb(struct sk_buff * skb,struct packet_type * pt_prev,struct net_device * orig_dev)1874 static inline int deliver_skb(struct sk_buff *skb,
1875 struct packet_type *pt_prev,
1876 struct net_device *orig_dev)
1877 {
1878 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1879 return -ENOMEM;
1880 refcount_inc(&skb->users);
1881 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1882 }
1883
deliver_ptype_list_skb(struct sk_buff * skb,struct packet_type ** pt,struct net_device * orig_dev,__be16 type,struct list_head * ptype_list)1884 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1885 struct packet_type **pt,
1886 struct net_device *orig_dev,
1887 __be16 type,
1888 struct list_head *ptype_list)
1889 {
1890 struct packet_type *ptype, *pt_prev = *pt;
1891
1892 list_for_each_entry_rcu(ptype, ptype_list, list) {
1893 if (ptype->type != type)
1894 continue;
1895 if (pt_prev)
1896 deliver_skb(skb, pt_prev, orig_dev);
1897 pt_prev = ptype;
1898 }
1899 *pt = pt_prev;
1900 }
1901
skb_loop_sk(struct packet_type * ptype,struct sk_buff * skb)1902 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1903 {
1904 if (!ptype->af_packet_priv || !skb->sk)
1905 return false;
1906
1907 if (ptype->id_match)
1908 return ptype->id_match(ptype, skb->sk);
1909 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1910 return true;
1911
1912 return false;
1913 }
1914
1915 /*
1916 * Support routine. Sends outgoing frames to any network
1917 * taps currently in use.
1918 */
1919
dev_queue_xmit_nit(struct sk_buff * skb,struct net_device * dev)1920 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1921 {
1922 struct packet_type *ptype;
1923 struct sk_buff *skb2 = NULL;
1924 struct packet_type *pt_prev = NULL;
1925 struct list_head *ptype_list = &ptype_all;
1926
1927 rcu_read_lock();
1928 again:
1929 list_for_each_entry_rcu(ptype, ptype_list, list) {
1930 /* Never send packets back to the socket
1931 * they originated from - MvS (miquels@drinkel.ow.org)
1932 */
1933 if (skb_loop_sk(ptype, skb))
1934 continue;
1935
1936 if (pt_prev) {
1937 deliver_skb(skb2, pt_prev, skb->dev);
1938 pt_prev = ptype;
1939 continue;
1940 }
1941
1942 /* need to clone skb, done only once */
1943 skb2 = skb_clone(skb, GFP_ATOMIC);
1944 if (!skb2)
1945 goto out_unlock;
1946
1947 net_timestamp_set(skb2);
1948
1949 /* skb->nh should be correctly
1950 * set by sender, so that the second statement is
1951 * just protection against buggy protocols.
1952 */
1953 skb_reset_mac_header(skb2);
1954
1955 if (skb_network_header(skb2) < skb2->data ||
1956 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1957 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1958 ntohs(skb2->protocol),
1959 dev->name);
1960 skb_reset_network_header(skb2);
1961 }
1962
1963 skb2->transport_header = skb2->network_header;
1964 skb2->pkt_type = PACKET_OUTGOING;
1965 pt_prev = ptype;
1966 }
1967
1968 if (ptype_list == &ptype_all) {
1969 ptype_list = &dev->ptype_all;
1970 goto again;
1971 }
1972 out_unlock:
1973 if (pt_prev) {
1974 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
1975 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1976 else
1977 kfree_skb(skb2);
1978 }
1979 rcu_read_unlock();
1980 }
1981 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1982
1983 /**
1984 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1985 * @dev: Network device
1986 * @txq: number of queues available
1987 *
1988 * If real_num_tx_queues is changed the tc mappings may no longer be
1989 * valid. To resolve this verify the tc mapping remains valid and if
1990 * not NULL the mapping. With no priorities mapping to this
1991 * offset/count pair it will no longer be used. In the worst case TC0
1992 * is invalid nothing can be done so disable priority mappings. If is
1993 * expected that drivers will fix this mapping if they can before
1994 * calling netif_set_real_num_tx_queues.
1995 */
netif_setup_tc(struct net_device * dev,unsigned int txq)1996 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1997 {
1998 int i;
1999 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2000
2001 /* If TC0 is invalidated disable TC mapping */
2002 if (tc->offset + tc->count > txq) {
2003 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2004 dev->num_tc = 0;
2005 return;
2006 }
2007
2008 /* Invalidated prio to tc mappings set to TC0 */
2009 for (i = 1; i < TC_BITMASK + 1; i++) {
2010 int q = netdev_get_prio_tc_map(dev, i);
2011
2012 tc = &dev->tc_to_txq[q];
2013 if (tc->offset + tc->count > txq) {
2014 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2015 i, q);
2016 netdev_set_prio_tc_map(dev, i, 0);
2017 }
2018 }
2019 }
2020
netdev_txq_to_tc(struct net_device * dev,unsigned int txq)2021 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2022 {
2023 if (dev->num_tc) {
2024 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2025 int i;
2026
2027 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2028 if ((txq - tc->offset) < tc->count)
2029 return i;
2030 }
2031
2032 return -1;
2033 }
2034
2035 return 0;
2036 }
2037
2038 #ifdef CONFIG_XPS
2039 static DEFINE_MUTEX(xps_map_mutex);
2040 #define xmap_dereference(P) \
2041 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2042
remove_xps_queue(struct xps_dev_maps * dev_maps,int tci,u16 index)2043 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2044 int tci, u16 index)
2045 {
2046 struct xps_map *map = NULL;
2047 int pos;
2048
2049 if (dev_maps)
2050 map = xmap_dereference(dev_maps->cpu_map[tci]);
2051 if (!map)
2052 return false;
2053
2054 for (pos = map->len; pos--;) {
2055 if (map->queues[pos] != index)
2056 continue;
2057
2058 if (map->len > 1) {
2059 map->queues[pos] = map->queues[--map->len];
2060 break;
2061 }
2062
2063 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2064 kfree_rcu(map, rcu);
2065 return false;
2066 }
2067
2068 return true;
2069 }
2070
remove_xps_queue_cpu(struct net_device * dev,struct xps_dev_maps * dev_maps,int cpu,u16 offset,u16 count)2071 static bool remove_xps_queue_cpu(struct net_device *dev,
2072 struct xps_dev_maps *dev_maps,
2073 int cpu, u16 offset, u16 count)
2074 {
2075 int num_tc = dev->num_tc ? : 1;
2076 bool active = false;
2077 int tci;
2078
2079 for (tci = cpu * num_tc; num_tc--; tci++) {
2080 int i, j;
2081
2082 for (i = count, j = offset; i--; j++) {
2083 if (!remove_xps_queue(dev_maps, tci, j))
2084 break;
2085 }
2086
2087 active |= i < 0;
2088 }
2089
2090 return active;
2091 }
2092
netif_reset_xps_queues(struct net_device * dev,u16 offset,u16 count)2093 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2094 u16 count)
2095 {
2096 struct xps_dev_maps *dev_maps;
2097 int cpu, i;
2098 bool active = false;
2099
2100 mutex_lock(&xps_map_mutex);
2101 dev_maps = xmap_dereference(dev->xps_maps);
2102
2103 if (!dev_maps)
2104 goto out_no_maps;
2105
2106 for_each_possible_cpu(cpu)
2107 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2108 offset, count);
2109
2110 if (!active) {
2111 RCU_INIT_POINTER(dev->xps_maps, NULL);
2112 kfree_rcu(dev_maps, rcu);
2113 }
2114
2115 for (i = offset + (count - 1); count--; i--)
2116 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2117 NUMA_NO_NODE);
2118
2119 out_no_maps:
2120 mutex_unlock(&xps_map_mutex);
2121 }
2122
netif_reset_xps_queues_gt(struct net_device * dev,u16 index)2123 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2124 {
2125 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2126 }
2127
expand_xps_map(struct xps_map * map,int cpu,u16 index)2128 static struct xps_map *expand_xps_map(struct xps_map *map,
2129 int cpu, u16 index)
2130 {
2131 struct xps_map *new_map;
2132 int alloc_len = XPS_MIN_MAP_ALLOC;
2133 int i, pos;
2134
2135 for (pos = 0; map && pos < map->len; pos++) {
2136 if (map->queues[pos] != index)
2137 continue;
2138 return map;
2139 }
2140
2141 /* Need to add queue to this CPU's existing map */
2142 if (map) {
2143 if (pos < map->alloc_len)
2144 return map;
2145
2146 alloc_len = map->alloc_len * 2;
2147 }
2148
2149 /* Need to allocate new map to store queue on this CPU's map */
2150 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2151 cpu_to_node(cpu));
2152 if (!new_map)
2153 return NULL;
2154
2155 for (i = 0; i < pos; i++)
2156 new_map->queues[i] = map->queues[i];
2157 new_map->alloc_len = alloc_len;
2158 new_map->len = pos;
2159
2160 return new_map;
2161 }
2162
netif_set_xps_queue(struct net_device * dev,const struct cpumask * mask,u16 index)2163 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2164 u16 index)
2165 {
2166 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2167 int i, cpu, tci, numa_node_id = -2;
2168 int maps_sz, num_tc = 1, tc = 0;
2169 struct xps_map *map, *new_map;
2170 bool active = false;
2171
2172 if (dev->num_tc) {
2173 num_tc = dev->num_tc;
2174 tc = netdev_txq_to_tc(dev, index);
2175 if (tc < 0)
2176 return -EINVAL;
2177 }
2178
2179 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2180 if (maps_sz < L1_CACHE_BYTES)
2181 maps_sz = L1_CACHE_BYTES;
2182
2183 mutex_lock(&xps_map_mutex);
2184
2185 dev_maps = xmap_dereference(dev->xps_maps);
2186
2187 /* allocate memory for queue storage */
2188 for_each_cpu_and(cpu, cpu_online_mask, mask) {
2189 if (!new_dev_maps)
2190 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2191 if (!new_dev_maps) {
2192 mutex_unlock(&xps_map_mutex);
2193 return -ENOMEM;
2194 }
2195
2196 tci = cpu * num_tc + tc;
2197 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2198 NULL;
2199
2200 map = expand_xps_map(map, cpu, index);
2201 if (!map)
2202 goto error;
2203
2204 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2205 }
2206
2207 if (!new_dev_maps)
2208 goto out_no_new_maps;
2209
2210 for_each_possible_cpu(cpu) {
2211 /* copy maps belonging to foreign traffic classes */
2212 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2213 /* fill in the new device map from the old device map */
2214 map = xmap_dereference(dev_maps->cpu_map[tci]);
2215 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2216 }
2217
2218 /* We need to explicitly update tci as prevous loop
2219 * could break out early if dev_maps is NULL.
2220 */
2221 tci = cpu * num_tc + tc;
2222
2223 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2224 /* add queue to CPU maps */
2225 int pos = 0;
2226
2227 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2228 while ((pos < map->len) && (map->queues[pos] != index))
2229 pos++;
2230
2231 if (pos == map->len)
2232 map->queues[map->len++] = index;
2233 #ifdef CONFIG_NUMA
2234 if (numa_node_id == -2)
2235 numa_node_id = cpu_to_node(cpu);
2236 else if (numa_node_id != cpu_to_node(cpu))
2237 numa_node_id = -1;
2238 #endif
2239 } else if (dev_maps) {
2240 /* fill in the new device map from the old device map */
2241 map = xmap_dereference(dev_maps->cpu_map[tci]);
2242 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2243 }
2244
2245 /* copy maps belonging to foreign traffic classes */
2246 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2247 /* fill in the new device map from the old device map */
2248 map = xmap_dereference(dev_maps->cpu_map[tci]);
2249 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2250 }
2251 }
2252
2253 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2254
2255 /* Cleanup old maps */
2256 if (!dev_maps)
2257 goto out_no_old_maps;
2258
2259 for_each_possible_cpu(cpu) {
2260 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2261 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2262 map = xmap_dereference(dev_maps->cpu_map[tci]);
2263 if (map && map != new_map)
2264 kfree_rcu(map, rcu);
2265 }
2266 }
2267
2268 kfree_rcu(dev_maps, rcu);
2269
2270 out_no_old_maps:
2271 dev_maps = new_dev_maps;
2272 active = true;
2273
2274 out_no_new_maps:
2275 /* update Tx queue numa node */
2276 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2277 (numa_node_id >= 0) ? numa_node_id :
2278 NUMA_NO_NODE);
2279
2280 if (!dev_maps)
2281 goto out_no_maps;
2282
2283 /* removes queue from unused CPUs */
2284 for_each_possible_cpu(cpu) {
2285 for (i = tc, tci = cpu * num_tc; i--; tci++)
2286 active |= remove_xps_queue(dev_maps, tci, index);
2287 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2288 active |= remove_xps_queue(dev_maps, tci, index);
2289 for (i = num_tc - tc, tci++; --i; tci++)
2290 active |= remove_xps_queue(dev_maps, tci, index);
2291 }
2292
2293 /* free map if not active */
2294 if (!active) {
2295 RCU_INIT_POINTER(dev->xps_maps, NULL);
2296 kfree_rcu(dev_maps, rcu);
2297 }
2298
2299 out_no_maps:
2300 mutex_unlock(&xps_map_mutex);
2301
2302 return 0;
2303 error:
2304 /* remove any maps that we added */
2305 for_each_possible_cpu(cpu) {
2306 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2307 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2308 map = dev_maps ?
2309 xmap_dereference(dev_maps->cpu_map[tci]) :
2310 NULL;
2311 if (new_map && new_map != map)
2312 kfree(new_map);
2313 }
2314 }
2315
2316 mutex_unlock(&xps_map_mutex);
2317
2318 kfree(new_dev_maps);
2319 return -ENOMEM;
2320 }
2321 EXPORT_SYMBOL(netif_set_xps_queue);
2322
2323 #endif
netdev_reset_tc(struct net_device * dev)2324 void netdev_reset_tc(struct net_device *dev)
2325 {
2326 #ifdef CONFIG_XPS
2327 netif_reset_xps_queues_gt(dev, 0);
2328 #endif
2329 dev->num_tc = 0;
2330 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2331 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2332 }
2333 EXPORT_SYMBOL(netdev_reset_tc);
2334
netdev_set_tc_queue(struct net_device * dev,u8 tc,u16 count,u16 offset)2335 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2336 {
2337 if (tc >= dev->num_tc)
2338 return -EINVAL;
2339
2340 #ifdef CONFIG_XPS
2341 netif_reset_xps_queues(dev, offset, count);
2342 #endif
2343 dev->tc_to_txq[tc].count = count;
2344 dev->tc_to_txq[tc].offset = offset;
2345 return 0;
2346 }
2347 EXPORT_SYMBOL(netdev_set_tc_queue);
2348
netdev_set_num_tc(struct net_device * dev,u8 num_tc)2349 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2350 {
2351 if (num_tc > TC_MAX_QUEUE)
2352 return -EINVAL;
2353
2354 #ifdef CONFIG_XPS
2355 netif_reset_xps_queues_gt(dev, 0);
2356 #endif
2357 dev->num_tc = num_tc;
2358 return 0;
2359 }
2360 EXPORT_SYMBOL(netdev_set_num_tc);
2361
2362 /*
2363 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2364 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2365 */
netif_set_real_num_tx_queues(struct net_device * dev,unsigned int txq)2366 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2367 {
2368 bool disabling;
2369 int rc;
2370
2371 disabling = txq < dev->real_num_tx_queues;
2372
2373 if (txq < 1 || txq > dev->num_tx_queues)
2374 return -EINVAL;
2375
2376 if (dev->reg_state == NETREG_REGISTERED ||
2377 dev->reg_state == NETREG_UNREGISTERING) {
2378 ASSERT_RTNL();
2379
2380 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2381 txq);
2382 if (rc)
2383 return rc;
2384
2385 if (dev->num_tc)
2386 netif_setup_tc(dev, txq);
2387
2388 dev->real_num_tx_queues = txq;
2389
2390 if (disabling) {
2391 synchronize_net();
2392 qdisc_reset_all_tx_gt(dev, txq);
2393 #ifdef CONFIG_XPS
2394 netif_reset_xps_queues_gt(dev, txq);
2395 #endif
2396 }
2397 } else {
2398 dev->real_num_tx_queues = txq;
2399 }
2400
2401 return 0;
2402 }
2403 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2404
2405 #ifdef CONFIG_SYSFS
2406 /**
2407 * netif_set_real_num_rx_queues - set actual number of RX queues used
2408 * @dev: Network device
2409 * @rxq: Actual number of RX queues
2410 *
2411 * This must be called either with the rtnl_lock held or before
2412 * registration of the net device. Returns 0 on success, or a
2413 * negative error code. If called before registration, it always
2414 * succeeds.
2415 */
netif_set_real_num_rx_queues(struct net_device * dev,unsigned int rxq)2416 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2417 {
2418 int rc;
2419
2420 if (rxq < 1 || rxq > dev->num_rx_queues)
2421 return -EINVAL;
2422
2423 if (dev->reg_state == NETREG_REGISTERED) {
2424 ASSERT_RTNL();
2425
2426 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2427 rxq);
2428 if (rc)
2429 return rc;
2430 }
2431
2432 dev->real_num_rx_queues = rxq;
2433 return 0;
2434 }
2435 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2436 #endif
2437
2438 /**
2439 * netif_get_num_default_rss_queues - default number of RSS queues
2440 *
2441 * This routine should set an upper limit on the number of RSS queues
2442 * used by default by multiqueue devices.
2443 */
netif_get_num_default_rss_queues(void)2444 int netif_get_num_default_rss_queues(void)
2445 {
2446 return is_kdump_kernel() ?
2447 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2448 }
2449 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2450
__netif_reschedule(struct Qdisc * q)2451 static void __netif_reschedule(struct Qdisc *q)
2452 {
2453 struct softnet_data *sd;
2454 unsigned long flags;
2455
2456 local_irq_save(flags);
2457 sd = this_cpu_ptr(&softnet_data);
2458 q->next_sched = NULL;
2459 *sd->output_queue_tailp = q;
2460 sd->output_queue_tailp = &q->next_sched;
2461 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2462 local_irq_restore(flags);
2463 }
2464
__netif_schedule(struct Qdisc * q)2465 void __netif_schedule(struct Qdisc *q)
2466 {
2467 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2468 __netif_reschedule(q);
2469 }
2470 EXPORT_SYMBOL(__netif_schedule);
2471
2472 struct dev_kfree_skb_cb {
2473 enum skb_free_reason reason;
2474 };
2475
get_kfree_skb_cb(const struct sk_buff * skb)2476 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2477 {
2478 return (struct dev_kfree_skb_cb *)skb->cb;
2479 }
2480
netif_schedule_queue(struct netdev_queue * txq)2481 void netif_schedule_queue(struct netdev_queue *txq)
2482 {
2483 rcu_read_lock();
2484 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2485 struct Qdisc *q = rcu_dereference(txq->qdisc);
2486
2487 __netif_schedule(q);
2488 }
2489 rcu_read_unlock();
2490 }
2491 EXPORT_SYMBOL(netif_schedule_queue);
2492
netif_tx_wake_queue(struct netdev_queue * dev_queue)2493 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2494 {
2495 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2496 struct Qdisc *q;
2497
2498 rcu_read_lock();
2499 q = rcu_dereference(dev_queue->qdisc);
2500 __netif_schedule(q);
2501 rcu_read_unlock();
2502 }
2503 }
2504 EXPORT_SYMBOL(netif_tx_wake_queue);
2505
__dev_kfree_skb_irq(struct sk_buff * skb,enum skb_free_reason reason)2506 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2507 {
2508 unsigned long flags;
2509
2510 if (unlikely(!skb))
2511 return;
2512
2513 if (likely(refcount_read(&skb->users) == 1)) {
2514 smp_rmb();
2515 refcount_set(&skb->users, 0);
2516 } else if (likely(!refcount_dec_and_test(&skb->users))) {
2517 return;
2518 }
2519 get_kfree_skb_cb(skb)->reason = reason;
2520 local_irq_save(flags);
2521 skb->next = __this_cpu_read(softnet_data.completion_queue);
2522 __this_cpu_write(softnet_data.completion_queue, skb);
2523 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2524 local_irq_restore(flags);
2525 }
2526 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2527
__dev_kfree_skb_any(struct sk_buff * skb,enum skb_free_reason reason)2528 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2529 {
2530 if (in_irq() || irqs_disabled())
2531 __dev_kfree_skb_irq(skb, reason);
2532 else
2533 dev_kfree_skb(skb);
2534 }
2535 EXPORT_SYMBOL(__dev_kfree_skb_any);
2536
2537
2538 /**
2539 * netif_device_detach - mark device as removed
2540 * @dev: network device
2541 *
2542 * Mark device as removed from system and therefore no longer available.
2543 */
netif_device_detach(struct net_device * dev)2544 void netif_device_detach(struct net_device *dev)
2545 {
2546 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2547 netif_running(dev)) {
2548 netif_tx_stop_all_queues(dev);
2549 }
2550 }
2551 EXPORT_SYMBOL(netif_device_detach);
2552
2553 /**
2554 * netif_device_attach - mark device as attached
2555 * @dev: network device
2556 *
2557 * Mark device as attached from system and restart if needed.
2558 */
netif_device_attach(struct net_device * dev)2559 void netif_device_attach(struct net_device *dev)
2560 {
2561 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2562 netif_running(dev)) {
2563 netif_tx_wake_all_queues(dev);
2564 __netdev_watchdog_up(dev);
2565 }
2566 }
2567 EXPORT_SYMBOL(netif_device_attach);
2568
2569 /*
2570 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2571 * to be used as a distribution range.
2572 */
__skb_tx_hash(const struct net_device * dev,struct sk_buff * skb,unsigned int num_tx_queues)2573 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2574 unsigned int num_tx_queues)
2575 {
2576 u32 hash;
2577 u16 qoffset = 0;
2578 u16 qcount = num_tx_queues;
2579
2580 if (skb_rx_queue_recorded(skb)) {
2581 hash = skb_get_rx_queue(skb);
2582 while (unlikely(hash >= num_tx_queues))
2583 hash -= num_tx_queues;
2584 return hash;
2585 }
2586
2587 if (dev->num_tc) {
2588 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2589
2590 qoffset = dev->tc_to_txq[tc].offset;
2591 qcount = dev->tc_to_txq[tc].count;
2592 }
2593
2594 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2595 }
2596 EXPORT_SYMBOL(__skb_tx_hash);
2597
skb_warn_bad_offload(const struct sk_buff * skb)2598 static void skb_warn_bad_offload(const struct sk_buff *skb)
2599 {
2600 static const netdev_features_t null_features;
2601 struct net_device *dev = skb->dev;
2602 const char *name = "";
2603
2604 if (!net_ratelimit())
2605 return;
2606
2607 if (dev) {
2608 if (dev->dev.parent)
2609 name = dev_driver_string(dev->dev.parent);
2610 else
2611 name = netdev_name(dev);
2612 }
2613 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2614 "gso_type=%d ip_summed=%d\n",
2615 name, dev ? &dev->features : &null_features,
2616 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2617 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2618 skb_shinfo(skb)->gso_type, skb->ip_summed);
2619 }
2620
2621 /*
2622 * Invalidate hardware checksum when packet is to be mangled, and
2623 * complete checksum manually on outgoing path.
2624 */
skb_checksum_help(struct sk_buff * skb)2625 int skb_checksum_help(struct sk_buff *skb)
2626 {
2627 __wsum csum;
2628 int ret = 0, offset;
2629
2630 if (skb->ip_summed == CHECKSUM_COMPLETE)
2631 goto out_set_summed;
2632
2633 if (unlikely(skb_shinfo(skb)->gso_size)) {
2634 skb_warn_bad_offload(skb);
2635 return -EINVAL;
2636 }
2637
2638 /* Before computing a checksum, we should make sure no frag could
2639 * be modified by an external entity : checksum could be wrong.
2640 */
2641 if (skb_has_shared_frag(skb)) {
2642 ret = __skb_linearize(skb);
2643 if (ret)
2644 goto out;
2645 }
2646
2647 offset = skb_checksum_start_offset(skb);
2648 BUG_ON(offset >= skb_headlen(skb));
2649 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2650
2651 offset += skb->csum_offset;
2652 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2653
2654 if (skb_cloned(skb) &&
2655 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2656 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2657 if (ret)
2658 goto out;
2659 }
2660
2661 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2662 out_set_summed:
2663 skb->ip_summed = CHECKSUM_NONE;
2664 out:
2665 return ret;
2666 }
2667 EXPORT_SYMBOL(skb_checksum_help);
2668
skb_crc32c_csum_help(struct sk_buff * skb)2669 int skb_crc32c_csum_help(struct sk_buff *skb)
2670 {
2671 __le32 crc32c_csum;
2672 int ret = 0, offset, start;
2673
2674 if (skb->ip_summed != CHECKSUM_PARTIAL)
2675 goto out;
2676
2677 if (unlikely(skb_is_gso(skb)))
2678 goto out;
2679
2680 /* Before computing a checksum, we should make sure no frag could
2681 * be modified by an external entity : checksum could be wrong.
2682 */
2683 if (unlikely(skb_has_shared_frag(skb))) {
2684 ret = __skb_linearize(skb);
2685 if (ret)
2686 goto out;
2687 }
2688 start = skb_checksum_start_offset(skb);
2689 offset = start + offsetof(struct sctphdr, checksum);
2690 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2691 ret = -EINVAL;
2692 goto out;
2693 }
2694 if (skb_cloned(skb) &&
2695 !skb_clone_writable(skb, offset + sizeof(__le32))) {
2696 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2697 if (ret)
2698 goto out;
2699 }
2700 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2701 skb->len - start, ~(__u32)0,
2702 crc32c_csum_stub));
2703 *(__le32 *)(skb->data + offset) = crc32c_csum;
2704 skb->ip_summed = CHECKSUM_NONE;
2705 skb->csum_not_inet = 0;
2706 out:
2707 return ret;
2708 }
2709
skb_network_protocol(struct sk_buff * skb,int * depth)2710 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2711 {
2712 __be16 type = skb->protocol;
2713
2714 /* Tunnel gso handlers can set protocol to ethernet. */
2715 if (type == htons(ETH_P_TEB)) {
2716 struct ethhdr *eth;
2717
2718 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2719 return 0;
2720
2721 eth = (struct ethhdr *)skb->data;
2722 type = eth->h_proto;
2723 }
2724
2725 return __vlan_get_protocol(skb, type, depth);
2726 }
2727
2728 /**
2729 * skb_mac_gso_segment - mac layer segmentation handler.
2730 * @skb: buffer to segment
2731 * @features: features for the output path (see dev->features)
2732 */
skb_mac_gso_segment(struct sk_buff * skb,netdev_features_t features)2733 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2734 netdev_features_t features)
2735 {
2736 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2737 struct packet_offload *ptype;
2738 int vlan_depth = skb->mac_len;
2739 __be16 type = skb_network_protocol(skb, &vlan_depth);
2740
2741 if (unlikely(!type))
2742 return ERR_PTR(-EINVAL);
2743
2744 __skb_pull(skb, vlan_depth);
2745
2746 rcu_read_lock();
2747 list_for_each_entry_rcu(ptype, &offload_base, list) {
2748 if (ptype->type == type && ptype->callbacks.gso_segment) {
2749 segs = ptype->callbacks.gso_segment(skb, features);
2750 break;
2751 }
2752 }
2753 rcu_read_unlock();
2754
2755 __skb_push(skb, skb->data - skb_mac_header(skb));
2756
2757 return segs;
2758 }
2759 EXPORT_SYMBOL(skb_mac_gso_segment);
2760
2761
2762 /* openvswitch calls this on rx path, so we need a different check.
2763 */
skb_needs_check(struct sk_buff * skb,bool tx_path)2764 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2765 {
2766 if (tx_path)
2767 return skb->ip_summed != CHECKSUM_PARTIAL &&
2768 skb->ip_summed != CHECKSUM_UNNECESSARY;
2769
2770 return skb->ip_summed == CHECKSUM_NONE;
2771 }
2772
2773 /**
2774 * __skb_gso_segment - Perform segmentation on skb.
2775 * @skb: buffer to segment
2776 * @features: features for the output path (see dev->features)
2777 * @tx_path: whether it is called in TX path
2778 *
2779 * This function segments the given skb and returns a list of segments.
2780 *
2781 * It may return NULL if the skb requires no segmentation. This is
2782 * only possible when GSO is used for verifying header integrity.
2783 *
2784 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2785 */
__skb_gso_segment(struct sk_buff * skb,netdev_features_t features,bool tx_path)2786 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2787 netdev_features_t features, bool tx_path)
2788 {
2789 struct sk_buff *segs;
2790
2791 if (unlikely(skb_needs_check(skb, tx_path))) {
2792 int err;
2793
2794 /* We're going to init ->check field in TCP or UDP header */
2795 err = skb_cow_head(skb, 0);
2796 if (err < 0)
2797 return ERR_PTR(err);
2798 }
2799
2800 /* Only report GSO partial support if it will enable us to
2801 * support segmentation on this frame without needing additional
2802 * work.
2803 */
2804 if (features & NETIF_F_GSO_PARTIAL) {
2805 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2806 struct net_device *dev = skb->dev;
2807
2808 partial_features |= dev->features & dev->gso_partial_features;
2809 if (!skb_gso_ok(skb, features | partial_features))
2810 features &= ~NETIF_F_GSO_PARTIAL;
2811 }
2812
2813 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2814 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2815
2816 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2817 SKB_GSO_CB(skb)->encap_level = 0;
2818
2819 skb_reset_mac_header(skb);
2820 skb_reset_mac_len(skb);
2821
2822 segs = skb_mac_gso_segment(skb, features);
2823
2824 if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
2825 skb_warn_bad_offload(skb);
2826
2827 return segs;
2828 }
2829 EXPORT_SYMBOL(__skb_gso_segment);
2830
2831 /* Take action when hardware reception checksum errors are detected. */
2832 #ifdef CONFIG_BUG
netdev_rx_csum_fault(struct net_device * dev)2833 void netdev_rx_csum_fault(struct net_device *dev)
2834 {
2835 if (net_ratelimit()) {
2836 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2837 dump_stack();
2838 }
2839 }
2840 EXPORT_SYMBOL(netdev_rx_csum_fault);
2841 #endif
2842
2843 /* Actually, we should eliminate this check as soon as we know, that:
2844 * 1. IOMMU is present and allows to map all the memory.
2845 * 2. No high memory really exists on this machine.
2846 */
2847
illegal_highdma(struct net_device * dev,struct sk_buff * skb)2848 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2849 {
2850 #ifdef CONFIG_HIGHMEM
2851 int i;
2852
2853 if (!(dev->features & NETIF_F_HIGHDMA)) {
2854 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2855 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2856
2857 if (PageHighMem(skb_frag_page(frag)))
2858 return 1;
2859 }
2860 }
2861
2862 if (PCI_DMA_BUS_IS_PHYS) {
2863 struct device *pdev = dev->dev.parent;
2864
2865 if (!pdev)
2866 return 0;
2867 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2868 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2869 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2870
2871 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2872 return 1;
2873 }
2874 }
2875 #endif
2876 return 0;
2877 }
2878
2879 /* If MPLS offload request, verify we are testing hardware MPLS features
2880 * instead of standard features for the netdev.
2881 */
2882 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
net_mpls_features(struct sk_buff * skb,netdev_features_t features,__be16 type)2883 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2884 netdev_features_t features,
2885 __be16 type)
2886 {
2887 if (eth_p_mpls(type))
2888 features &= skb->dev->mpls_features;
2889
2890 return features;
2891 }
2892 #else
net_mpls_features(struct sk_buff * skb,netdev_features_t features,__be16 type)2893 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2894 netdev_features_t features,
2895 __be16 type)
2896 {
2897 return features;
2898 }
2899 #endif
2900
harmonize_features(struct sk_buff * skb,netdev_features_t features)2901 static netdev_features_t harmonize_features(struct sk_buff *skb,
2902 netdev_features_t features)
2903 {
2904 int tmp;
2905 __be16 type;
2906
2907 type = skb_network_protocol(skb, &tmp);
2908 features = net_mpls_features(skb, features, type);
2909
2910 if (skb->ip_summed != CHECKSUM_NONE &&
2911 !can_checksum_protocol(features, type)) {
2912 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2913 }
2914 if (illegal_highdma(skb->dev, skb))
2915 features &= ~NETIF_F_SG;
2916
2917 return features;
2918 }
2919
passthru_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2920 netdev_features_t passthru_features_check(struct sk_buff *skb,
2921 struct net_device *dev,
2922 netdev_features_t features)
2923 {
2924 return features;
2925 }
2926 EXPORT_SYMBOL(passthru_features_check);
2927
dflt_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2928 static netdev_features_t dflt_features_check(struct sk_buff *skb,
2929 struct net_device *dev,
2930 netdev_features_t features)
2931 {
2932 return vlan_features_check(skb, features);
2933 }
2934
gso_features_check(const struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2935 static netdev_features_t gso_features_check(const struct sk_buff *skb,
2936 struct net_device *dev,
2937 netdev_features_t features)
2938 {
2939 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2940
2941 if (gso_segs > dev->gso_max_segs)
2942 return features & ~NETIF_F_GSO_MASK;
2943
2944 /* Support for GSO partial features requires software
2945 * intervention before we can actually process the packets
2946 * so we need to strip support for any partial features now
2947 * and we can pull them back in after we have partially
2948 * segmented the frame.
2949 */
2950 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2951 features &= ~dev->gso_partial_features;
2952
2953 /* Make sure to clear the IPv4 ID mangling feature if the
2954 * IPv4 header has the potential to be fragmented.
2955 */
2956 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2957 struct iphdr *iph = skb->encapsulation ?
2958 inner_ip_hdr(skb) : ip_hdr(skb);
2959
2960 if (!(iph->frag_off & htons(IP_DF)))
2961 features &= ~NETIF_F_TSO_MANGLEID;
2962 }
2963
2964 return features;
2965 }
2966
netif_skb_features(struct sk_buff * skb)2967 netdev_features_t netif_skb_features(struct sk_buff *skb)
2968 {
2969 struct net_device *dev = skb->dev;
2970 netdev_features_t features = dev->features;
2971
2972 if (skb_is_gso(skb))
2973 features = gso_features_check(skb, dev, features);
2974
2975 /* If encapsulation offload request, verify we are testing
2976 * hardware encapsulation features instead of standard
2977 * features for the netdev
2978 */
2979 if (skb->encapsulation)
2980 features &= dev->hw_enc_features;
2981
2982 if (skb_vlan_tagged(skb))
2983 features = netdev_intersect_features(features,
2984 dev->vlan_features |
2985 NETIF_F_HW_VLAN_CTAG_TX |
2986 NETIF_F_HW_VLAN_STAG_TX);
2987
2988 if (dev->netdev_ops->ndo_features_check)
2989 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2990 features);
2991 else
2992 features &= dflt_features_check(skb, dev, features);
2993
2994 return harmonize_features(skb, features);
2995 }
2996 EXPORT_SYMBOL(netif_skb_features);
2997
xmit_one(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq,bool more)2998 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2999 struct netdev_queue *txq, bool more)
3000 {
3001 unsigned int len;
3002 int rc;
3003
3004 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
3005 dev_queue_xmit_nit(skb, dev);
3006
3007 len = skb->len;
3008 trace_net_dev_start_xmit(skb, dev);
3009 rc = netdev_start_xmit(skb, dev, txq, more);
3010 trace_net_dev_xmit(skb, rc, dev, len);
3011
3012 return rc;
3013 }
3014
dev_hard_start_xmit(struct sk_buff * first,struct net_device * dev,struct netdev_queue * txq,int * ret)3015 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3016 struct netdev_queue *txq, int *ret)
3017 {
3018 struct sk_buff *skb = first;
3019 int rc = NETDEV_TX_OK;
3020
3021 while (skb) {
3022 struct sk_buff *next = skb->next;
3023
3024 skb->next = NULL;
3025 rc = xmit_one(skb, dev, txq, next != NULL);
3026 if (unlikely(!dev_xmit_complete(rc))) {
3027 skb->next = next;
3028 goto out;
3029 }
3030
3031 skb = next;
3032 if (netif_tx_queue_stopped(txq) && skb) {
3033 rc = NETDEV_TX_BUSY;
3034 break;
3035 }
3036 }
3037
3038 out:
3039 *ret = rc;
3040 return skb;
3041 }
3042
validate_xmit_vlan(struct sk_buff * skb,netdev_features_t features)3043 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3044 netdev_features_t features)
3045 {
3046 if (skb_vlan_tag_present(skb) &&
3047 !vlan_hw_offload_capable(features, skb->vlan_proto))
3048 skb = __vlan_hwaccel_push_inside(skb);
3049 return skb;
3050 }
3051
skb_csum_hwoffload_help(struct sk_buff * skb,const netdev_features_t features)3052 int skb_csum_hwoffload_help(struct sk_buff *skb,
3053 const netdev_features_t features)
3054 {
3055 if (unlikely(skb->csum_not_inet))
3056 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3057 skb_crc32c_csum_help(skb);
3058
3059 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3060 }
3061 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3062
validate_xmit_skb(struct sk_buff * skb,struct net_device * dev)3063 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
3064 {
3065 netdev_features_t features;
3066
3067 features = netif_skb_features(skb);
3068 skb = validate_xmit_vlan(skb, features);
3069 if (unlikely(!skb))
3070 goto out_null;
3071
3072 if (netif_needs_gso(skb, features)) {
3073 struct sk_buff *segs;
3074
3075 segs = skb_gso_segment(skb, features);
3076 if (IS_ERR(segs)) {
3077 goto out_kfree_skb;
3078 } else if (segs) {
3079 consume_skb(skb);
3080 skb = segs;
3081 }
3082 } else {
3083 if (skb_needs_linearize(skb, features) &&
3084 __skb_linearize(skb))
3085 goto out_kfree_skb;
3086
3087 if (validate_xmit_xfrm(skb, features))
3088 goto out_kfree_skb;
3089
3090 /* If packet is not checksummed and device does not
3091 * support checksumming for this protocol, complete
3092 * checksumming here.
3093 */
3094 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3095 if (skb->encapsulation)
3096 skb_set_inner_transport_header(skb,
3097 skb_checksum_start_offset(skb));
3098 else
3099 skb_set_transport_header(skb,
3100 skb_checksum_start_offset(skb));
3101 if (skb_csum_hwoffload_help(skb, features))
3102 goto out_kfree_skb;
3103 }
3104 }
3105
3106 return skb;
3107
3108 out_kfree_skb:
3109 kfree_skb(skb);
3110 out_null:
3111 atomic_long_inc(&dev->tx_dropped);
3112 return NULL;
3113 }
3114
validate_xmit_skb_list(struct sk_buff * skb,struct net_device * dev)3115 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3116 {
3117 struct sk_buff *next, *head = NULL, *tail;
3118
3119 for (; skb != NULL; skb = next) {
3120 next = skb->next;
3121 skb->next = NULL;
3122
3123 /* in case skb wont be segmented, point to itself */
3124 skb->prev = skb;
3125
3126 skb = validate_xmit_skb(skb, dev);
3127 if (!skb)
3128 continue;
3129
3130 if (!head)
3131 head = skb;
3132 else
3133 tail->next = skb;
3134 /* If skb was segmented, skb->prev points to
3135 * the last segment. If not, it still contains skb.
3136 */
3137 tail = skb->prev;
3138 }
3139 return head;
3140 }
3141 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3142
qdisc_pkt_len_init(struct sk_buff * skb)3143 static void qdisc_pkt_len_init(struct sk_buff *skb)
3144 {
3145 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3146
3147 qdisc_skb_cb(skb)->pkt_len = skb->len;
3148
3149 /* To get more precise estimation of bytes sent on wire,
3150 * we add to pkt_len the headers size of all segments
3151 */
3152 if (shinfo->gso_size) {
3153 unsigned int hdr_len;
3154 u16 gso_segs = shinfo->gso_segs;
3155
3156 /* mac layer + network layer */
3157 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3158
3159 /* + transport layer */
3160 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3161 const struct tcphdr *th;
3162 struct tcphdr _tcphdr;
3163
3164 th = skb_header_pointer(skb, skb_transport_offset(skb),
3165 sizeof(_tcphdr), &_tcphdr);
3166 if (likely(th))
3167 hdr_len += __tcp_hdrlen(th);
3168 } else {
3169 struct udphdr _udphdr;
3170
3171 if (skb_header_pointer(skb, skb_transport_offset(skb),
3172 sizeof(_udphdr), &_udphdr))
3173 hdr_len += sizeof(struct udphdr);
3174 }
3175
3176 if (shinfo->gso_type & SKB_GSO_DODGY)
3177 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3178 shinfo->gso_size);
3179
3180 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3181 }
3182 }
3183
__dev_xmit_skb(struct sk_buff * skb,struct Qdisc * q,struct net_device * dev,struct netdev_queue * txq)3184 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3185 struct net_device *dev,
3186 struct netdev_queue *txq)
3187 {
3188 spinlock_t *root_lock = qdisc_lock(q);
3189 struct sk_buff *to_free = NULL;
3190 bool contended;
3191 int rc;
3192
3193 qdisc_calculate_pkt_len(skb, q);
3194 /*
3195 * Heuristic to force contended enqueues to serialize on a
3196 * separate lock before trying to get qdisc main lock.
3197 * This permits qdisc->running owner to get the lock more
3198 * often and dequeue packets faster.
3199 */
3200 contended = qdisc_is_running(q);
3201 if (unlikely(contended))
3202 spin_lock(&q->busylock);
3203
3204 spin_lock(root_lock);
3205 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3206 __qdisc_drop(skb, &to_free);
3207 rc = NET_XMIT_DROP;
3208 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3209 qdisc_run_begin(q)) {
3210 /*
3211 * This is a work-conserving queue; there are no old skbs
3212 * waiting to be sent out; and the qdisc is not running -
3213 * xmit the skb directly.
3214 */
3215
3216 qdisc_bstats_update(q, skb);
3217
3218 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3219 if (unlikely(contended)) {
3220 spin_unlock(&q->busylock);
3221 contended = false;
3222 }
3223 __qdisc_run(q);
3224 } else
3225 qdisc_run_end(q);
3226
3227 rc = NET_XMIT_SUCCESS;
3228 } else {
3229 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3230 if (qdisc_run_begin(q)) {
3231 if (unlikely(contended)) {
3232 spin_unlock(&q->busylock);
3233 contended = false;
3234 }
3235 __qdisc_run(q);
3236 }
3237 }
3238 spin_unlock(root_lock);
3239 if (unlikely(to_free))
3240 kfree_skb_list(to_free);
3241 if (unlikely(contended))
3242 spin_unlock(&q->busylock);
3243 return rc;
3244 }
3245
3246 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
skb_update_prio(struct sk_buff * skb)3247 static void skb_update_prio(struct sk_buff *skb)
3248 {
3249 const struct netprio_map *map;
3250 const struct sock *sk;
3251 unsigned int prioidx;
3252
3253 if (skb->priority)
3254 return;
3255 map = rcu_dereference_bh(skb->dev->priomap);
3256 if (!map)
3257 return;
3258 sk = skb_to_full_sk(skb);
3259 if (!sk)
3260 return;
3261
3262 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3263
3264 if (prioidx < map->priomap_len)
3265 skb->priority = map->priomap[prioidx];
3266 }
3267 #else
3268 #define skb_update_prio(skb)
3269 #endif
3270
3271 DEFINE_PER_CPU(int, xmit_recursion);
3272 EXPORT_SYMBOL(xmit_recursion);
3273
3274 /**
3275 * dev_loopback_xmit - loop back @skb
3276 * @net: network namespace this loopback is happening in
3277 * @sk: sk needed to be a netfilter okfn
3278 * @skb: buffer to transmit
3279 */
dev_loopback_xmit(struct net * net,struct sock * sk,struct sk_buff * skb)3280 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3281 {
3282 skb_reset_mac_header(skb);
3283 __skb_pull(skb, skb_network_offset(skb));
3284 skb->pkt_type = PACKET_LOOPBACK;
3285 skb->ip_summed = CHECKSUM_UNNECESSARY;
3286 WARN_ON(!skb_dst(skb));
3287 skb_dst_force(skb);
3288 netif_rx_ni(skb);
3289 return 0;
3290 }
3291 EXPORT_SYMBOL(dev_loopback_xmit);
3292
3293 #ifdef CONFIG_NET_EGRESS
3294 static struct sk_buff *
sch_handle_egress(struct sk_buff * skb,int * ret,struct net_device * dev)3295 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3296 {
3297 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3298 struct tcf_result cl_res;
3299
3300 if (!cl)
3301 return skb;
3302
3303 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3304 qdisc_bstats_cpu_update(cl->q, skb);
3305
3306 switch (tcf_classify(skb, cl, &cl_res, false)) {
3307 case TC_ACT_OK:
3308 case TC_ACT_RECLASSIFY:
3309 skb->tc_index = TC_H_MIN(cl_res.classid);
3310 break;
3311 case TC_ACT_SHOT:
3312 qdisc_qstats_cpu_drop(cl->q);
3313 *ret = NET_XMIT_DROP;
3314 kfree_skb(skb);
3315 return NULL;
3316 case TC_ACT_STOLEN:
3317 case TC_ACT_QUEUED:
3318 case TC_ACT_TRAP:
3319 *ret = NET_XMIT_SUCCESS;
3320 consume_skb(skb);
3321 return NULL;
3322 case TC_ACT_REDIRECT:
3323 /* No need to push/pop skb's mac_header here on egress! */
3324 skb_do_redirect(skb);
3325 *ret = NET_XMIT_SUCCESS;
3326 return NULL;
3327 default:
3328 break;
3329 }
3330
3331 return skb;
3332 }
3333 #endif /* CONFIG_NET_EGRESS */
3334
get_xps_queue(struct net_device * dev,struct sk_buff * skb)3335 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3336 {
3337 #ifdef CONFIG_XPS
3338 struct xps_dev_maps *dev_maps;
3339 struct xps_map *map;
3340 int queue_index = -1;
3341
3342 rcu_read_lock();
3343 dev_maps = rcu_dereference(dev->xps_maps);
3344 if (dev_maps) {
3345 unsigned int tci = skb->sender_cpu - 1;
3346
3347 if (dev->num_tc) {
3348 tci *= dev->num_tc;
3349 tci += netdev_get_prio_tc_map(dev, skb->priority);
3350 }
3351
3352 map = rcu_dereference(dev_maps->cpu_map[tci]);
3353 if (map) {
3354 if (map->len == 1)
3355 queue_index = map->queues[0];
3356 else
3357 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3358 map->len)];
3359 if (unlikely(queue_index >= dev->real_num_tx_queues))
3360 queue_index = -1;
3361 }
3362 }
3363 rcu_read_unlock();
3364
3365 return queue_index;
3366 #else
3367 return -1;
3368 #endif
3369 }
3370
__netdev_pick_tx(struct net_device * dev,struct sk_buff * skb)3371 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3372 {
3373 struct sock *sk = skb->sk;
3374 int queue_index = sk_tx_queue_get(sk);
3375
3376 if (queue_index < 0 || skb->ooo_okay ||
3377 queue_index >= dev->real_num_tx_queues) {
3378 int new_index = get_xps_queue(dev, skb);
3379
3380 if (new_index < 0)
3381 new_index = skb_tx_hash(dev, skb);
3382
3383 if (queue_index != new_index && sk &&
3384 sk_fullsock(sk) &&
3385 rcu_access_pointer(sk->sk_dst_cache))
3386 sk_tx_queue_set(sk, new_index);
3387
3388 queue_index = new_index;
3389 }
3390
3391 return queue_index;
3392 }
3393
netdev_pick_tx(struct net_device * dev,struct sk_buff * skb,void * accel_priv)3394 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3395 struct sk_buff *skb,
3396 void *accel_priv)
3397 {
3398 int queue_index = 0;
3399
3400 #ifdef CONFIG_XPS
3401 u32 sender_cpu = skb->sender_cpu - 1;
3402
3403 if (sender_cpu >= (u32)NR_CPUS)
3404 skb->sender_cpu = raw_smp_processor_id() + 1;
3405 #endif
3406
3407 if (dev->real_num_tx_queues != 1) {
3408 const struct net_device_ops *ops = dev->netdev_ops;
3409
3410 if (ops->ndo_select_queue)
3411 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3412 __netdev_pick_tx);
3413 else
3414 queue_index = __netdev_pick_tx(dev, skb);
3415
3416 if (!accel_priv)
3417 queue_index = netdev_cap_txqueue(dev, queue_index);
3418 }
3419
3420 skb_set_queue_mapping(skb, queue_index);
3421 return netdev_get_tx_queue(dev, queue_index);
3422 }
3423
3424 /**
3425 * __dev_queue_xmit - transmit a buffer
3426 * @skb: buffer to transmit
3427 * @accel_priv: private data used for L2 forwarding offload
3428 *
3429 * Queue a buffer for transmission to a network device. The caller must
3430 * have set the device and priority and built the buffer before calling
3431 * this function. The function can be called from an interrupt.
3432 *
3433 * A negative errno code is returned on a failure. A success does not
3434 * guarantee the frame will be transmitted as it may be dropped due
3435 * to congestion or traffic shaping.
3436 *
3437 * -----------------------------------------------------------------------------------
3438 * I notice this method can also return errors from the queue disciplines,
3439 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3440 * be positive.
3441 *
3442 * Regardless of the return value, the skb is consumed, so it is currently
3443 * difficult to retry a send to this method. (You can bump the ref count
3444 * before sending to hold a reference for retry if you are careful.)
3445 *
3446 * When calling this method, interrupts MUST be enabled. This is because
3447 * the BH enable code must have IRQs enabled so that it will not deadlock.
3448 * --BLG
3449 */
__dev_queue_xmit(struct sk_buff * skb,void * accel_priv)3450 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3451 {
3452 struct net_device *dev = skb->dev;
3453 struct netdev_queue *txq;
3454 struct Qdisc *q;
3455 int rc = -ENOMEM;
3456
3457 skb_reset_mac_header(skb);
3458
3459 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3460 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3461
3462 /* Disable soft irqs for various locks below. Also
3463 * stops preemption for RCU.
3464 */
3465 rcu_read_lock_bh();
3466
3467 skb_update_prio(skb);
3468
3469 qdisc_pkt_len_init(skb);
3470 #ifdef CONFIG_NET_CLS_ACT
3471 skb->tc_at_ingress = 0;
3472 # ifdef CONFIG_NET_EGRESS
3473 if (static_key_false(&egress_needed)) {
3474 skb = sch_handle_egress(skb, &rc, dev);
3475 if (!skb)
3476 goto out;
3477 }
3478 # endif
3479 #endif
3480 /* If device/qdisc don't need skb->dst, release it right now while
3481 * its hot in this cpu cache.
3482 */
3483 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3484 skb_dst_drop(skb);
3485 else
3486 skb_dst_force(skb);
3487
3488 txq = netdev_pick_tx(dev, skb, accel_priv);
3489 q = rcu_dereference_bh(txq->qdisc);
3490
3491 trace_net_dev_queue(skb);
3492 if (q->enqueue) {
3493 rc = __dev_xmit_skb(skb, q, dev, txq);
3494 goto out;
3495 }
3496
3497 /* The device has no queue. Common case for software devices:
3498 * loopback, all the sorts of tunnels...
3499
3500 * Really, it is unlikely that netif_tx_lock protection is necessary
3501 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
3502 * counters.)
3503 * However, it is possible, that they rely on protection
3504 * made by us here.
3505
3506 * Check this and shot the lock. It is not prone from deadlocks.
3507 *Either shot noqueue qdisc, it is even simpler 8)
3508 */
3509 if (dev->flags & IFF_UP) {
3510 int cpu = smp_processor_id(); /* ok because BHs are off */
3511
3512 if (txq->xmit_lock_owner != cpu) {
3513 if (unlikely(__this_cpu_read(xmit_recursion) >
3514 XMIT_RECURSION_LIMIT))
3515 goto recursion_alert;
3516
3517 skb = validate_xmit_skb(skb, dev);
3518 if (!skb)
3519 goto out;
3520
3521 HARD_TX_LOCK(dev, txq, cpu);
3522
3523 if (!netif_xmit_stopped(txq)) {
3524 __this_cpu_inc(xmit_recursion);
3525 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3526 __this_cpu_dec(xmit_recursion);
3527 if (dev_xmit_complete(rc)) {
3528 HARD_TX_UNLOCK(dev, txq);
3529 goto out;
3530 }
3531 }
3532 HARD_TX_UNLOCK(dev, txq);
3533 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3534 dev->name);
3535 } else {
3536 /* Recursion is detected! It is possible,
3537 * unfortunately
3538 */
3539 recursion_alert:
3540 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3541 dev->name);
3542 }
3543 }
3544
3545 rc = -ENETDOWN;
3546 rcu_read_unlock_bh();
3547
3548 atomic_long_inc(&dev->tx_dropped);
3549 kfree_skb_list(skb);
3550 return rc;
3551 out:
3552 rcu_read_unlock_bh();
3553 return rc;
3554 }
3555
dev_queue_xmit(struct sk_buff * skb)3556 int dev_queue_xmit(struct sk_buff *skb)
3557 {
3558 return __dev_queue_xmit(skb, NULL);
3559 }
3560 EXPORT_SYMBOL(dev_queue_xmit);
3561
dev_queue_xmit_accel(struct sk_buff * skb,void * accel_priv)3562 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3563 {
3564 return __dev_queue_xmit(skb, accel_priv);
3565 }
3566 EXPORT_SYMBOL(dev_queue_xmit_accel);
3567
3568
3569 /*************************************************************************
3570 * Receiver routines
3571 *************************************************************************/
3572
3573 int netdev_max_backlog __read_mostly = 1000;
3574 EXPORT_SYMBOL(netdev_max_backlog);
3575
3576 int netdev_tstamp_prequeue __read_mostly = 1;
3577 int netdev_budget __read_mostly = 300;
3578 unsigned int __read_mostly netdev_budget_usecs = 2000;
3579 int weight_p __read_mostly = 64; /* old backlog weight */
3580 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
3581 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
3582 int dev_rx_weight __read_mostly = 64;
3583 int dev_tx_weight __read_mostly = 64;
3584
3585 /* Called with irq disabled */
____napi_schedule(struct softnet_data * sd,struct napi_struct * napi)3586 static inline void ____napi_schedule(struct softnet_data *sd,
3587 struct napi_struct *napi)
3588 {
3589 list_add_tail(&napi->poll_list, &sd->poll_list);
3590 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3591 }
3592
3593 #ifdef CONFIG_RPS
3594
3595 /* One global table that all flow-based protocols share. */
3596 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3597 EXPORT_SYMBOL(rps_sock_flow_table);
3598 u32 rps_cpu_mask __read_mostly;
3599 EXPORT_SYMBOL(rps_cpu_mask);
3600
3601 struct static_key rps_needed __read_mostly;
3602 EXPORT_SYMBOL(rps_needed);
3603 struct static_key rfs_needed __read_mostly;
3604 EXPORT_SYMBOL(rfs_needed);
3605
3606 static struct rps_dev_flow *
set_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow * rflow,u16 next_cpu)3607 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3608 struct rps_dev_flow *rflow, u16 next_cpu)
3609 {
3610 if (next_cpu < nr_cpu_ids) {
3611 #ifdef CONFIG_RFS_ACCEL
3612 struct netdev_rx_queue *rxqueue;
3613 struct rps_dev_flow_table *flow_table;
3614 struct rps_dev_flow *old_rflow;
3615 u32 flow_id;
3616 u16 rxq_index;
3617 int rc;
3618
3619 /* Should we steer this flow to a different hardware queue? */
3620 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3621 !(dev->features & NETIF_F_NTUPLE))
3622 goto out;
3623 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3624 if (rxq_index == skb_get_rx_queue(skb))
3625 goto out;
3626
3627 rxqueue = dev->_rx + rxq_index;
3628 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3629 if (!flow_table)
3630 goto out;
3631 flow_id = skb_get_hash(skb) & flow_table->mask;
3632 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3633 rxq_index, flow_id);
3634 if (rc < 0)
3635 goto out;
3636 old_rflow = rflow;
3637 rflow = &flow_table->flows[flow_id];
3638 rflow->filter = rc;
3639 if (old_rflow->filter == rflow->filter)
3640 old_rflow->filter = RPS_NO_FILTER;
3641 out:
3642 #endif
3643 rflow->last_qtail =
3644 per_cpu(softnet_data, next_cpu).input_queue_head;
3645 }
3646
3647 rflow->cpu = next_cpu;
3648 return rflow;
3649 }
3650
3651 /*
3652 * get_rps_cpu is called from netif_receive_skb and returns the target
3653 * CPU from the RPS map of the receiving queue for a given skb.
3654 * rcu_read_lock must be held on entry.
3655 */
get_rps_cpu(struct net_device * dev,struct sk_buff * skb,struct rps_dev_flow ** rflowp)3656 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3657 struct rps_dev_flow **rflowp)
3658 {
3659 const struct rps_sock_flow_table *sock_flow_table;
3660 struct netdev_rx_queue *rxqueue = dev->_rx;
3661 struct rps_dev_flow_table *flow_table;
3662 struct rps_map *map;
3663 int cpu = -1;
3664 u32 tcpu;
3665 u32 hash;
3666
3667 if (skb_rx_queue_recorded(skb)) {
3668 u16 index = skb_get_rx_queue(skb);
3669
3670 if (unlikely(index >= dev->real_num_rx_queues)) {
3671 WARN_ONCE(dev->real_num_rx_queues > 1,
3672 "%s received packet on queue %u, but number "
3673 "of RX queues is %u\n",
3674 dev->name, index, dev->real_num_rx_queues);
3675 goto done;
3676 }
3677 rxqueue += index;
3678 }
3679
3680 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3681
3682 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3683 map = rcu_dereference(rxqueue->rps_map);
3684 if (!flow_table && !map)
3685 goto done;
3686
3687 skb_reset_network_header(skb);
3688 hash = skb_get_hash(skb);
3689 if (!hash)
3690 goto done;
3691
3692 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3693 if (flow_table && sock_flow_table) {
3694 struct rps_dev_flow *rflow;
3695 u32 next_cpu;
3696 u32 ident;
3697
3698 /* First check into global flow table if there is a match */
3699 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3700 if ((ident ^ hash) & ~rps_cpu_mask)
3701 goto try_rps;
3702
3703 next_cpu = ident & rps_cpu_mask;
3704
3705 /* OK, now we know there is a match,
3706 * we can look at the local (per receive queue) flow table
3707 */
3708 rflow = &flow_table->flows[hash & flow_table->mask];
3709 tcpu = rflow->cpu;
3710
3711 /*
3712 * If the desired CPU (where last recvmsg was done) is
3713 * different from current CPU (one in the rx-queue flow
3714 * table entry), switch if one of the following holds:
3715 * - Current CPU is unset (>= nr_cpu_ids).
3716 * - Current CPU is offline.
3717 * - The current CPU's queue tail has advanced beyond the
3718 * last packet that was enqueued using this table entry.
3719 * This guarantees that all previous packets for the flow
3720 * have been dequeued, thus preserving in order delivery.
3721 */
3722 if (unlikely(tcpu != next_cpu) &&
3723 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3724 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3725 rflow->last_qtail)) >= 0)) {
3726 tcpu = next_cpu;
3727 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3728 }
3729
3730 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3731 *rflowp = rflow;
3732 cpu = tcpu;
3733 goto done;
3734 }
3735 }
3736
3737 try_rps:
3738
3739 if (map) {
3740 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3741 if (cpu_online(tcpu)) {
3742 cpu = tcpu;
3743 goto done;
3744 }
3745 }
3746
3747 done:
3748 return cpu;
3749 }
3750
3751 #ifdef CONFIG_RFS_ACCEL
3752
3753 /**
3754 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3755 * @dev: Device on which the filter was set
3756 * @rxq_index: RX queue index
3757 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3758 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3759 *
3760 * Drivers that implement ndo_rx_flow_steer() should periodically call
3761 * this function for each installed filter and remove the filters for
3762 * which it returns %true.
3763 */
rps_may_expire_flow(struct net_device * dev,u16 rxq_index,u32 flow_id,u16 filter_id)3764 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3765 u32 flow_id, u16 filter_id)
3766 {
3767 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3768 struct rps_dev_flow_table *flow_table;
3769 struct rps_dev_flow *rflow;
3770 bool expire = true;
3771 unsigned int cpu;
3772
3773 rcu_read_lock();
3774 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3775 if (flow_table && flow_id <= flow_table->mask) {
3776 rflow = &flow_table->flows[flow_id];
3777 cpu = ACCESS_ONCE(rflow->cpu);
3778 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3779 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3780 rflow->last_qtail) <
3781 (int)(10 * flow_table->mask)))
3782 expire = false;
3783 }
3784 rcu_read_unlock();
3785 return expire;
3786 }
3787 EXPORT_SYMBOL(rps_may_expire_flow);
3788
3789 #endif /* CONFIG_RFS_ACCEL */
3790
3791 /* Called from hardirq (IPI) context */
rps_trigger_softirq(void * data)3792 static void rps_trigger_softirq(void *data)
3793 {
3794 struct softnet_data *sd = data;
3795
3796 ____napi_schedule(sd, &sd->backlog);
3797 sd->received_rps++;
3798 }
3799
3800 #endif /* CONFIG_RPS */
3801
3802 /*
3803 * Check if this softnet_data structure is another cpu one
3804 * If yes, queue it to our IPI list and return 1
3805 * If no, return 0
3806 */
rps_ipi_queued(struct softnet_data * sd)3807 static int rps_ipi_queued(struct softnet_data *sd)
3808 {
3809 #ifdef CONFIG_RPS
3810 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3811
3812 if (sd != mysd) {
3813 sd->rps_ipi_next = mysd->rps_ipi_list;
3814 mysd->rps_ipi_list = sd;
3815
3816 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3817 return 1;
3818 }
3819 #endif /* CONFIG_RPS */
3820 return 0;
3821 }
3822
3823 #ifdef CONFIG_NET_FLOW_LIMIT
3824 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3825 #endif
3826
skb_flow_limit(struct sk_buff * skb,unsigned int qlen)3827 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3828 {
3829 #ifdef CONFIG_NET_FLOW_LIMIT
3830 struct sd_flow_limit *fl;
3831 struct softnet_data *sd;
3832 unsigned int old_flow, new_flow;
3833
3834 if (qlen < (netdev_max_backlog >> 1))
3835 return false;
3836
3837 sd = this_cpu_ptr(&softnet_data);
3838
3839 rcu_read_lock();
3840 fl = rcu_dereference(sd->flow_limit);
3841 if (fl) {
3842 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3843 old_flow = fl->history[fl->history_head];
3844 fl->history[fl->history_head] = new_flow;
3845
3846 fl->history_head++;
3847 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3848
3849 if (likely(fl->buckets[old_flow]))
3850 fl->buckets[old_flow]--;
3851
3852 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3853 fl->count++;
3854 rcu_read_unlock();
3855 return true;
3856 }
3857 }
3858 rcu_read_unlock();
3859 #endif
3860 return false;
3861 }
3862
3863 /*
3864 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3865 * queue (may be a remote CPU queue).
3866 */
enqueue_to_backlog(struct sk_buff * skb,int cpu,unsigned int * qtail)3867 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3868 unsigned int *qtail)
3869 {
3870 struct softnet_data *sd;
3871 unsigned long flags;
3872 unsigned int qlen;
3873
3874 sd = &per_cpu(softnet_data, cpu);
3875
3876 local_irq_save(flags);
3877
3878 rps_lock(sd);
3879 if (!netif_running(skb->dev))
3880 goto drop;
3881 qlen = skb_queue_len(&sd->input_pkt_queue);
3882 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3883 if (qlen) {
3884 enqueue:
3885 __skb_queue_tail(&sd->input_pkt_queue, skb);
3886 input_queue_tail_incr_save(sd, qtail);
3887 rps_unlock(sd);
3888 local_irq_restore(flags);
3889 return NET_RX_SUCCESS;
3890 }
3891
3892 /* Schedule NAPI for backlog device
3893 * We can use non atomic operation since we own the queue lock
3894 */
3895 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3896 if (!rps_ipi_queued(sd))
3897 ____napi_schedule(sd, &sd->backlog);
3898 }
3899 goto enqueue;
3900 }
3901
3902 drop:
3903 sd->dropped++;
3904 rps_unlock(sd);
3905
3906 local_irq_restore(flags);
3907
3908 atomic_long_inc(&skb->dev->rx_dropped);
3909 kfree_skb(skb);
3910 return NET_RX_DROP;
3911 }
3912
netif_receive_generic_xdp(struct sk_buff * skb,struct bpf_prog * xdp_prog)3913 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
3914 struct bpf_prog *xdp_prog)
3915 {
3916 struct xdp_buff xdp;
3917 u32 act = XDP_DROP;
3918 void *orig_data;
3919 int hlen, off;
3920 u32 mac_len;
3921
3922 /* Reinjected packets coming from act_mirred or similar should
3923 * not get XDP generic processing.
3924 */
3925 if (skb_cloned(skb))
3926 return XDP_PASS;
3927
3928 if (skb_linearize(skb))
3929 goto do_drop;
3930
3931 /* The XDP program wants to see the packet starting at the MAC
3932 * header.
3933 */
3934 mac_len = skb->data - skb_mac_header(skb);
3935 hlen = skb_headlen(skb) + mac_len;
3936 xdp.data = skb->data - mac_len;
3937 xdp.data_end = xdp.data + hlen;
3938 xdp.data_hard_start = skb->data - skb_headroom(skb);
3939 orig_data = xdp.data;
3940
3941 act = bpf_prog_run_xdp(xdp_prog, &xdp);
3942
3943 off = xdp.data - orig_data;
3944 if (off > 0)
3945 __skb_pull(skb, off);
3946 else if (off < 0)
3947 __skb_push(skb, -off);
3948 skb->mac_header += off;
3949
3950 switch (act) {
3951 case XDP_REDIRECT:
3952 case XDP_TX:
3953 __skb_push(skb, mac_len);
3954 /* fall through */
3955 case XDP_PASS:
3956 break;
3957
3958 default:
3959 bpf_warn_invalid_xdp_action(act);
3960 /* fall through */
3961 case XDP_ABORTED:
3962 trace_xdp_exception(skb->dev, xdp_prog, act);
3963 /* fall through */
3964 case XDP_DROP:
3965 do_drop:
3966 kfree_skb(skb);
3967 break;
3968 }
3969
3970 return act;
3971 }
3972
3973 /* When doing generic XDP we have to bypass the qdisc layer and the
3974 * network taps in order to match in-driver-XDP behavior.
3975 */
generic_xdp_tx(struct sk_buff * skb,struct bpf_prog * xdp_prog)3976 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
3977 {
3978 struct net_device *dev = skb->dev;
3979 struct netdev_queue *txq;
3980 bool free_skb = true;
3981 int cpu, rc;
3982
3983 txq = netdev_pick_tx(dev, skb, NULL);
3984 cpu = smp_processor_id();
3985 HARD_TX_LOCK(dev, txq, cpu);
3986 if (!netif_xmit_stopped(txq)) {
3987 rc = netdev_start_xmit(skb, dev, txq, 0);
3988 if (dev_xmit_complete(rc))
3989 free_skb = false;
3990 }
3991 HARD_TX_UNLOCK(dev, txq);
3992 if (free_skb) {
3993 trace_xdp_exception(dev, xdp_prog, XDP_TX);
3994 kfree_skb(skb);
3995 }
3996 }
3997 EXPORT_SYMBOL_GPL(generic_xdp_tx);
3998
3999 static struct static_key generic_xdp_needed __read_mostly;
4000
do_xdp_generic(struct bpf_prog * xdp_prog,struct sk_buff * skb)4001 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4002 {
4003 if (xdp_prog) {
4004 u32 act = netif_receive_generic_xdp(skb, xdp_prog);
4005 int err;
4006
4007 if (act != XDP_PASS) {
4008 switch (act) {
4009 case XDP_REDIRECT:
4010 err = xdp_do_generic_redirect(skb->dev, skb,
4011 xdp_prog);
4012 if (err)
4013 goto out_redir;
4014 /* fallthru to submit skb */
4015 case XDP_TX:
4016 generic_xdp_tx(skb, xdp_prog);
4017 break;
4018 }
4019 return XDP_DROP;
4020 }
4021 }
4022 return XDP_PASS;
4023 out_redir:
4024 kfree_skb(skb);
4025 return XDP_DROP;
4026 }
4027 EXPORT_SYMBOL_GPL(do_xdp_generic);
4028
netif_rx_internal(struct sk_buff * skb)4029 static int netif_rx_internal(struct sk_buff *skb)
4030 {
4031 int ret;
4032
4033 net_timestamp_check(netdev_tstamp_prequeue, skb);
4034
4035 trace_netif_rx(skb);
4036
4037 if (static_key_false(&generic_xdp_needed)) {
4038 int ret;
4039
4040 preempt_disable();
4041 rcu_read_lock();
4042 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4043 rcu_read_unlock();
4044 preempt_enable();
4045
4046 /* Consider XDP consuming the packet a success from
4047 * the netdev point of view we do not want to count
4048 * this as an error.
4049 */
4050 if (ret != XDP_PASS)
4051 return NET_RX_SUCCESS;
4052 }
4053
4054 #ifdef CONFIG_RPS
4055 if (static_key_false(&rps_needed)) {
4056 struct rps_dev_flow voidflow, *rflow = &voidflow;
4057 int cpu;
4058
4059 preempt_disable();
4060 rcu_read_lock();
4061
4062 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4063 if (cpu < 0)
4064 cpu = smp_processor_id();
4065
4066 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4067
4068 rcu_read_unlock();
4069 preempt_enable();
4070 } else
4071 #endif
4072 {
4073 unsigned int qtail;
4074
4075 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4076 put_cpu();
4077 }
4078 return ret;
4079 }
4080
4081 /**
4082 * netif_rx - post buffer to the network code
4083 * @skb: buffer to post
4084 *
4085 * This function receives a packet from a device driver and queues it for
4086 * the upper (protocol) levels to process. It always succeeds. The buffer
4087 * may be dropped during processing for congestion control or by the
4088 * protocol layers.
4089 *
4090 * return values:
4091 * NET_RX_SUCCESS (no congestion)
4092 * NET_RX_DROP (packet was dropped)
4093 *
4094 */
4095
netif_rx(struct sk_buff * skb)4096 int netif_rx(struct sk_buff *skb)
4097 {
4098 trace_netif_rx_entry(skb);
4099
4100 return netif_rx_internal(skb);
4101 }
4102 EXPORT_SYMBOL(netif_rx);
4103
netif_rx_ni(struct sk_buff * skb)4104 int netif_rx_ni(struct sk_buff *skb)
4105 {
4106 int err;
4107
4108 trace_netif_rx_ni_entry(skb);
4109
4110 preempt_disable();
4111 err = netif_rx_internal(skb);
4112 if (local_softirq_pending())
4113 do_softirq();
4114 preempt_enable();
4115
4116 return err;
4117 }
4118 EXPORT_SYMBOL(netif_rx_ni);
4119
net_tx_action(struct softirq_action * h)4120 static __latent_entropy void net_tx_action(struct softirq_action *h)
4121 {
4122 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4123
4124 if (sd->completion_queue) {
4125 struct sk_buff *clist;
4126
4127 local_irq_disable();
4128 clist = sd->completion_queue;
4129 sd->completion_queue = NULL;
4130 local_irq_enable();
4131
4132 while (clist) {
4133 struct sk_buff *skb = clist;
4134
4135 clist = clist->next;
4136
4137 WARN_ON(refcount_read(&skb->users));
4138 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4139 trace_consume_skb(skb);
4140 else
4141 trace_kfree_skb(skb, net_tx_action);
4142
4143 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4144 __kfree_skb(skb);
4145 else
4146 __kfree_skb_defer(skb);
4147 }
4148
4149 __kfree_skb_flush();
4150 }
4151
4152 if (sd->output_queue) {
4153 struct Qdisc *head;
4154
4155 local_irq_disable();
4156 head = sd->output_queue;
4157 sd->output_queue = NULL;
4158 sd->output_queue_tailp = &sd->output_queue;
4159 local_irq_enable();
4160
4161 while (head) {
4162 struct Qdisc *q = head;
4163 spinlock_t *root_lock;
4164
4165 head = head->next_sched;
4166
4167 root_lock = qdisc_lock(q);
4168 spin_lock(root_lock);
4169 /* We need to make sure head->next_sched is read
4170 * before clearing __QDISC_STATE_SCHED
4171 */
4172 smp_mb__before_atomic();
4173 clear_bit(__QDISC_STATE_SCHED, &q->state);
4174 qdisc_run(q);
4175 spin_unlock(root_lock);
4176 }
4177 }
4178 }
4179
4180 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4181 /* This hook is defined here for ATM LANE */
4182 int (*br_fdb_test_addr_hook)(struct net_device *dev,
4183 unsigned char *addr) __read_mostly;
4184 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
4185 #endif
4186
4187 static inline struct sk_buff *
sch_handle_ingress(struct sk_buff * skb,struct packet_type ** pt_prev,int * ret,struct net_device * orig_dev)4188 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4189 struct net_device *orig_dev)
4190 {
4191 #ifdef CONFIG_NET_CLS_ACT
4192 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
4193 struct tcf_result cl_res;
4194
4195 /* If there's at least one ingress present somewhere (so
4196 * we get here via enabled static key), remaining devices
4197 * that are not configured with an ingress qdisc will bail
4198 * out here.
4199 */
4200 if (!cl)
4201 return skb;
4202 if (*pt_prev) {
4203 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4204 *pt_prev = NULL;
4205 }
4206
4207 qdisc_skb_cb(skb)->pkt_len = skb->len;
4208 skb->tc_at_ingress = 1;
4209 qdisc_bstats_cpu_update(cl->q, skb);
4210
4211 switch (tcf_classify(skb, cl, &cl_res, false)) {
4212 case TC_ACT_OK:
4213 case TC_ACT_RECLASSIFY:
4214 skb->tc_index = TC_H_MIN(cl_res.classid);
4215 break;
4216 case TC_ACT_SHOT:
4217 qdisc_qstats_cpu_drop(cl->q);
4218 kfree_skb(skb);
4219 return NULL;
4220 case TC_ACT_STOLEN:
4221 case TC_ACT_QUEUED:
4222 case TC_ACT_TRAP:
4223 consume_skb(skb);
4224 return NULL;
4225 case TC_ACT_REDIRECT:
4226 /* skb_mac_header check was done by cls/act_bpf, so
4227 * we can safely push the L2 header back before
4228 * redirecting to another netdev
4229 */
4230 __skb_push(skb, skb->mac_len);
4231 skb_do_redirect(skb);
4232 return NULL;
4233 default:
4234 break;
4235 }
4236 #endif /* CONFIG_NET_CLS_ACT */
4237 return skb;
4238 }
4239
4240 /**
4241 * netdev_is_rx_handler_busy - check if receive handler is registered
4242 * @dev: device to check
4243 *
4244 * Check if a receive handler is already registered for a given device.
4245 * Return true if there one.
4246 *
4247 * The caller must hold the rtnl_mutex.
4248 */
netdev_is_rx_handler_busy(struct net_device * dev)4249 bool netdev_is_rx_handler_busy(struct net_device *dev)
4250 {
4251 ASSERT_RTNL();
4252 return dev && rtnl_dereference(dev->rx_handler);
4253 }
4254 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4255
4256 /**
4257 * netdev_rx_handler_register - register receive handler
4258 * @dev: device to register a handler for
4259 * @rx_handler: receive handler to register
4260 * @rx_handler_data: data pointer that is used by rx handler
4261 *
4262 * Register a receive handler for a device. This handler will then be
4263 * called from __netif_receive_skb. A negative errno code is returned
4264 * on a failure.
4265 *
4266 * The caller must hold the rtnl_mutex.
4267 *
4268 * For a general description of rx_handler, see enum rx_handler_result.
4269 */
netdev_rx_handler_register(struct net_device * dev,rx_handler_func_t * rx_handler,void * rx_handler_data)4270 int netdev_rx_handler_register(struct net_device *dev,
4271 rx_handler_func_t *rx_handler,
4272 void *rx_handler_data)
4273 {
4274 if (netdev_is_rx_handler_busy(dev))
4275 return -EBUSY;
4276
4277 /* Note: rx_handler_data must be set before rx_handler */
4278 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4279 rcu_assign_pointer(dev->rx_handler, rx_handler);
4280
4281 return 0;
4282 }
4283 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4284
4285 /**
4286 * netdev_rx_handler_unregister - unregister receive handler
4287 * @dev: device to unregister a handler from
4288 *
4289 * Unregister a receive handler from a device.
4290 *
4291 * The caller must hold the rtnl_mutex.
4292 */
netdev_rx_handler_unregister(struct net_device * dev)4293 void netdev_rx_handler_unregister(struct net_device *dev)
4294 {
4295
4296 ASSERT_RTNL();
4297 RCU_INIT_POINTER(dev->rx_handler, NULL);
4298 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4299 * section has a guarantee to see a non NULL rx_handler_data
4300 * as well.
4301 */
4302 synchronize_net();
4303 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4304 }
4305 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4306
4307 /*
4308 * Limit the use of PFMEMALLOC reserves to those protocols that implement
4309 * the special handling of PFMEMALLOC skbs.
4310 */
skb_pfmemalloc_protocol(struct sk_buff * skb)4311 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4312 {
4313 switch (skb->protocol) {
4314 case htons(ETH_P_ARP):
4315 case htons(ETH_P_IP):
4316 case htons(ETH_P_IPV6):
4317 case htons(ETH_P_8021Q):
4318 case htons(ETH_P_8021AD):
4319 return true;
4320 default:
4321 return false;
4322 }
4323 }
4324
nf_ingress(struct sk_buff * skb,struct packet_type ** pt_prev,int * ret,struct net_device * orig_dev)4325 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4326 int *ret, struct net_device *orig_dev)
4327 {
4328 #ifdef CONFIG_NETFILTER_INGRESS
4329 if (nf_hook_ingress_active(skb)) {
4330 int ingress_retval;
4331
4332 if (*pt_prev) {
4333 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4334 *pt_prev = NULL;
4335 }
4336
4337 rcu_read_lock();
4338 ingress_retval = nf_hook_ingress(skb);
4339 rcu_read_unlock();
4340 return ingress_retval;
4341 }
4342 #endif /* CONFIG_NETFILTER_INGRESS */
4343 return 0;
4344 }
4345
__netif_receive_skb_core(struct sk_buff * skb,bool pfmemalloc)4346 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4347 {
4348 struct packet_type *ptype, *pt_prev;
4349 rx_handler_func_t *rx_handler;
4350 struct net_device *orig_dev;
4351 bool deliver_exact = false;
4352 int ret = NET_RX_DROP;
4353 __be16 type;
4354
4355 net_timestamp_check(!netdev_tstamp_prequeue, skb);
4356
4357 trace_netif_receive_skb(skb);
4358
4359 orig_dev = skb->dev;
4360
4361 skb_reset_network_header(skb);
4362 if (!skb_transport_header_was_set(skb))
4363 skb_reset_transport_header(skb);
4364 skb_reset_mac_len(skb);
4365
4366 pt_prev = NULL;
4367
4368 another_round:
4369 skb->skb_iif = skb->dev->ifindex;
4370
4371 __this_cpu_inc(softnet_data.processed);
4372
4373 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4374 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4375 skb = skb_vlan_untag(skb);
4376 if (unlikely(!skb))
4377 goto out;
4378 }
4379
4380 if (skb_skip_tc_classify(skb))
4381 goto skip_classify;
4382
4383 if (pfmemalloc)
4384 goto skip_taps;
4385
4386 list_for_each_entry_rcu(ptype, &ptype_all, list) {
4387 if (pt_prev)
4388 ret = deliver_skb(skb, pt_prev, orig_dev);
4389 pt_prev = ptype;
4390 }
4391
4392 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4393 if (pt_prev)
4394 ret = deliver_skb(skb, pt_prev, orig_dev);
4395 pt_prev = ptype;
4396 }
4397
4398 skip_taps:
4399 #ifdef CONFIG_NET_INGRESS
4400 if (static_key_false(&ingress_needed)) {
4401 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4402 if (!skb)
4403 goto out;
4404
4405 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4406 goto out;
4407 }
4408 #endif
4409 skb_reset_tc(skb);
4410 skip_classify:
4411 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4412 goto drop;
4413
4414 if (skb_vlan_tag_present(skb)) {
4415 if (pt_prev) {
4416 ret = deliver_skb(skb, pt_prev, orig_dev);
4417 pt_prev = NULL;
4418 }
4419 if (vlan_do_receive(&skb))
4420 goto another_round;
4421 else if (unlikely(!skb))
4422 goto out;
4423 }
4424
4425 rx_handler = rcu_dereference(skb->dev->rx_handler);
4426 if (rx_handler) {
4427 if (pt_prev) {
4428 ret = deliver_skb(skb, pt_prev, orig_dev);
4429 pt_prev = NULL;
4430 }
4431 switch (rx_handler(&skb)) {
4432 case RX_HANDLER_CONSUMED:
4433 ret = NET_RX_SUCCESS;
4434 goto out;
4435 case RX_HANDLER_ANOTHER:
4436 goto another_round;
4437 case RX_HANDLER_EXACT:
4438 deliver_exact = true;
4439 case RX_HANDLER_PASS:
4440 break;
4441 default:
4442 BUG();
4443 }
4444 }
4445
4446 if (unlikely(skb_vlan_tag_present(skb))) {
4447 if (skb_vlan_tag_get_id(skb))
4448 skb->pkt_type = PACKET_OTHERHOST;
4449 /* Note: we might in the future use prio bits
4450 * and set skb->priority like in vlan_do_receive()
4451 * For the time being, just ignore Priority Code Point
4452 */
4453 skb->vlan_tci = 0;
4454 }
4455
4456 type = skb->protocol;
4457
4458 /* deliver only exact match when indicated */
4459 if (likely(!deliver_exact)) {
4460 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4461 &ptype_base[ntohs(type) &
4462 PTYPE_HASH_MASK]);
4463 }
4464
4465 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4466 &orig_dev->ptype_specific);
4467
4468 if (unlikely(skb->dev != orig_dev)) {
4469 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4470 &skb->dev->ptype_specific);
4471 }
4472
4473 if (pt_prev) {
4474 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
4475 goto drop;
4476 else
4477 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4478 } else {
4479 drop:
4480 if (!deliver_exact)
4481 atomic_long_inc(&skb->dev->rx_dropped);
4482 else
4483 atomic_long_inc(&skb->dev->rx_nohandler);
4484 kfree_skb(skb);
4485 /* Jamal, now you will not able to escape explaining
4486 * me how you were going to use this. :-)
4487 */
4488 ret = NET_RX_DROP;
4489 }
4490
4491 out:
4492 return ret;
4493 }
4494
__netif_receive_skb(struct sk_buff * skb)4495 static int __netif_receive_skb(struct sk_buff *skb)
4496 {
4497 int ret;
4498
4499 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4500 unsigned int noreclaim_flag;
4501
4502 /*
4503 * PFMEMALLOC skbs are special, they should
4504 * - be delivered to SOCK_MEMALLOC sockets only
4505 * - stay away from userspace
4506 * - have bounded memory usage
4507 *
4508 * Use PF_MEMALLOC as this saves us from propagating the allocation
4509 * context down to all allocation sites.
4510 */
4511 noreclaim_flag = memalloc_noreclaim_save();
4512 ret = __netif_receive_skb_core(skb, true);
4513 memalloc_noreclaim_restore(noreclaim_flag);
4514 } else
4515 ret = __netif_receive_skb_core(skb, false);
4516
4517 return ret;
4518 }
4519
generic_xdp_install(struct net_device * dev,struct netdev_xdp * xdp)4520 static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
4521 {
4522 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
4523 struct bpf_prog *new = xdp->prog;
4524 int ret = 0;
4525
4526 switch (xdp->command) {
4527 case XDP_SETUP_PROG:
4528 rcu_assign_pointer(dev->xdp_prog, new);
4529 if (old)
4530 bpf_prog_put(old);
4531
4532 if (old && !new) {
4533 static_key_slow_dec(&generic_xdp_needed);
4534 } else if (new && !old) {
4535 static_key_slow_inc(&generic_xdp_needed);
4536 dev_disable_lro(dev);
4537 }
4538 break;
4539
4540 case XDP_QUERY_PROG:
4541 xdp->prog_attached = !!old;
4542 xdp->prog_id = old ? old->aux->id : 0;
4543 break;
4544
4545 default:
4546 ret = -EINVAL;
4547 break;
4548 }
4549
4550 return ret;
4551 }
4552
netif_receive_skb_internal(struct sk_buff * skb)4553 static int netif_receive_skb_internal(struct sk_buff *skb)
4554 {
4555 int ret;
4556
4557 net_timestamp_check(netdev_tstamp_prequeue, skb);
4558
4559 if (skb_defer_rx_timestamp(skb))
4560 return NET_RX_SUCCESS;
4561
4562 if (static_key_false(&generic_xdp_needed)) {
4563 int ret;
4564
4565 preempt_disable();
4566 rcu_read_lock();
4567 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4568 rcu_read_unlock();
4569 preempt_enable();
4570
4571 if (ret != XDP_PASS)
4572 return NET_RX_DROP;
4573 }
4574
4575 rcu_read_lock();
4576 #ifdef CONFIG_RPS
4577 if (static_key_false(&rps_needed)) {
4578 struct rps_dev_flow voidflow, *rflow = &voidflow;
4579 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4580
4581 if (cpu >= 0) {
4582 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4583 rcu_read_unlock();
4584 return ret;
4585 }
4586 }
4587 #endif
4588 ret = __netif_receive_skb(skb);
4589 rcu_read_unlock();
4590 return ret;
4591 }
4592
4593 /**
4594 * netif_receive_skb - process receive buffer from network
4595 * @skb: buffer to process
4596 *
4597 * netif_receive_skb() is the main receive data processing function.
4598 * It always succeeds. The buffer may be dropped during processing
4599 * for congestion control or by the protocol layers.
4600 *
4601 * This function may only be called from softirq context and interrupts
4602 * should be enabled.
4603 *
4604 * Return values (usually ignored):
4605 * NET_RX_SUCCESS: no congestion
4606 * NET_RX_DROP: packet was dropped
4607 */
netif_receive_skb(struct sk_buff * skb)4608 int netif_receive_skb(struct sk_buff *skb)
4609 {
4610 trace_netif_receive_skb_entry(skb);
4611
4612 return netif_receive_skb_internal(skb);
4613 }
4614 EXPORT_SYMBOL(netif_receive_skb);
4615
4616 DEFINE_PER_CPU(struct work_struct, flush_works);
4617
4618 /* Network device is going away, flush any packets still pending */
flush_backlog(struct work_struct * work)4619 static void flush_backlog(struct work_struct *work)
4620 {
4621 struct sk_buff *skb, *tmp;
4622 struct softnet_data *sd;
4623
4624 local_bh_disable();
4625 sd = this_cpu_ptr(&softnet_data);
4626
4627 local_irq_disable();
4628 rps_lock(sd);
4629 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4630 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4631 __skb_unlink(skb, &sd->input_pkt_queue);
4632 kfree_skb(skb);
4633 input_queue_head_incr(sd);
4634 }
4635 }
4636 rps_unlock(sd);
4637 local_irq_enable();
4638
4639 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4640 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4641 __skb_unlink(skb, &sd->process_queue);
4642 kfree_skb(skb);
4643 input_queue_head_incr(sd);
4644 }
4645 }
4646 local_bh_enable();
4647 }
4648
flush_all_backlogs(void)4649 static void flush_all_backlogs(void)
4650 {
4651 unsigned int cpu;
4652
4653 get_online_cpus();
4654
4655 for_each_online_cpu(cpu)
4656 queue_work_on(cpu, system_highpri_wq,
4657 per_cpu_ptr(&flush_works, cpu));
4658
4659 for_each_online_cpu(cpu)
4660 flush_work(per_cpu_ptr(&flush_works, cpu));
4661
4662 put_online_cpus();
4663 }
4664
napi_gro_complete(struct sk_buff * skb)4665 static int napi_gro_complete(struct sk_buff *skb)
4666 {
4667 struct packet_offload *ptype;
4668 __be16 type = skb->protocol;
4669 struct list_head *head = &offload_base;
4670 int err = -ENOENT;
4671
4672 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4673
4674 if (NAPI_GRO_CB(skb)->count == 1) {
4675 skb_shinfo(skb)->gso_size = 0;
4676 goto out;
4677 }
4678
4679 rcu_read_lock();
4680 list_for_each_entry_rcu(ptype, head, list) {
4681 if (ptype->type != type || !ptype->callbacks.gro_complete)
4682 continue;
4683
4684 err = ptype->callbacks.gro_complete(skb, 0);
4685 break;
4686 }
4687 rcu_read_unlock();
4688
4689 if (err) {
4690 WARN_ON(&ptype->list == head);
4691 kfree_skb(skb);
4692 return NET_RX_SUCCESS;
4693 }
4694
4695 out:
4696 return netif_receive_skb_internal(skb);
4697 }
4698
4699 /* napi->gro_list contains packets ordered by age.
4700 * youngest packets at the head of it.
4701 * Complete skbs in reverse order to reduce latencies.
4702 */
napi_gro_flush(struct napi_struct * napi,bool flush_old)4703 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4704 {
4705 struct sk_buff *skb, *prev = NULL;
4706
4707 /* scan list and build reverse chain */
4708 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4709 skb->prev = prev;
4710 prev = skb;
4711 }
4712
4713 for (skb = prev; skb; skb = prev) {
4714 skb->next = NULL;
4715
4716 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4717 return;
4718
4719 prev = skb->prev;
4720 napi_gro_complete(skb);
4721 napi->gro_count--;
4722 }
4723
4724 napi->gro_list = NULL;
4725 }
4726 EXPORT_SYMBOL(napi_gro_flush);
4727
gro_list_prepare(struct napi_struct * napi,struct sk_buff * skb)4728 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4729 {
4730 struct sk_buff *p;
4731 unsigned int maclen = skb->dev->hard_header_len;
4732 u32 hash = skb_get_hash_raw(skb);
4733
4734 for (p = napi->gro_list; p; p = p->next) {
4735 unsigned long diffs;
4736
4737 NAPI_GRO_CB(p)->flush = 0;
4738
4739 if (hash != skb_get_hash_raw(p)) {
4740 NAPI_GRO_CB(p)->same_flow = 0;
4741 continue;
4742 }
4743
4744 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4745 diffs |= p->vlan_tci ^ skb->vlan_tci;
4746 diffs |= skb_metadata_dst_cmp(p, skb);
4747 if (maclen == ETH_HLEN)
4748 diffs |= compare_ether_header(skb_mac_header(p),
4749 skb_mac_header(skb));
4750 else if (!diffs)
4751 diffs = memcmp(skb_mac_header(p),
4752 skb_mac_header(skb),
4753 maclen);
4754 NAPI_GRO_CB(p)->same_flow = !diffs;
4755 }
4756 }
4757
skb_gro_reset_offset(struct sk_buff * skb)4758 static void skb_gro_reset_offset(struct sk_buff *skb)
4759 {
4760 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4761 const skb_frag_t *frag0 = &pinfo->frags[0];
4762
4763 NAPI_GRO_CB(skb)->data_offset = 0;
4764 NAPI_GRO_CB(skb)->frag0 = NULL;
4765 NAPI_GRO_CB(skb)->frag0_len = 0;
4766
4767 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4768 pinfo->nr_frags &&
4769 !PageHighMem(skb_frag_page(frag0))) {
4770 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4771 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4772 skb_frag_size(frag0),
4773 skb->end - skb->tail);
4774 }
4775 }
4776
gro_pull_from_frag0(struct sk_buff * skb,int grow)4777 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4778 {
4779 struct skb_shared_info *pinfo = skb_shinfo(skb);
4780
4781 BUG_ON(skb->end - skb->tail < grow);
4782
4783 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4784
4785 skb->data_len -= grow;
4786 skb->tail += grow;
4787
4788 pinfo->frags[0].page_offset += grow;
4789 skb_frag_size_sub(&pinfo->frags[0], grow);
4790
4791 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4792 skb_frag_unref(skb, 0);
4793 memmove(pinfo->frags, pinfo->frags + 1,
4794 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4795 }
4796 }
4797
dev_gro_receive(struct napi_struct * napi,struct sk_buff * skb)4798 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4799 {
4800 struct sk_buff **pp = NULL;
4801 struct packet_offload *ptype;
4802 __be16 type = skb->protocol;
4803 struct list_head *head = &offload_base;
4804 int same_flow;
4805 enum gro_result ret;
4806 int grow;
4807
4808 if (netif_elide_gro(skb->dev))
4809 goto normal;
4810
4811 gro_list_prepare(napi, skb);
4812
4813 rcu_read_lock();
4814 list_for_each_entry_rcu(ptype, head, list) {
4815 if (ptype->type != type || !ptype->callbacks.gro_receive)
4816 continue;
4817
4818 skb_set_network_header(skb, skb_gro_offset(skb));
4819 skb_reset_mac_len(skb);
4820 NAPI_GRO_CB(skb)->same_flow = 0;
4821 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
4822 NAPI_GRO_CB(skb)->free = 0;
4823 NAPI_GRO_CB(skb)->encap_mark = 0;
4824 NAPI_GRO_CB(skb)->recursion_counter = 0;
4825 NAPI_GRO_CB(skb)->is_fou = 0;
4826 NAPI_GRO_CB(skb)->is_atomic = 1;
4827 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4828
4829 /* Setup for GRO checksum validation */
4830 switch (skb->ip_summed) {
4831 case CHECKSUM_COMPLETE:
4832 NAPI_GRO_CB(skb)->csum = skb->csum;
4833 NAPI_GRO_CB(skb)->csum_valid = 1;
4834 NAPI_GRO_CB(skb)->csum_cnt = 0;
4835 break;
4836 case CHECKSUM_UNNECESSARY:
4837 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4838 NAPI_GRO_CB(skb)->csum_valid = 0;
4839 break;
4840 default:
4841 NAPI_GRO_CB(skb)->csum_cnt = 0;
4842 NAPI_GRO_CB(skb)->csum_valid = 0;
4843 }
4844
4845 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4846 break;
4847 }
4848 rcu_read_unlock();
4849
4850 if (&ptype->list == head)
4851 goto normal;
4852
4853 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4854 ret = GRO_CONSUMED;
4855 goto ok;
4856 }
4857
4858 same_flow = NAPI_GRO_CB(skb)->same_flow;
4859 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4860
4861 if (pp) {
4862 struct sk_buff *nskb = *pp;
4863
4864 *pp = nskb->next;
4865 nskb->next = NULL;
4866 napi_gro_complete(nskb);
4867 napi->gro_count--;
4868 }
4869
4870 if (same_flow)
4871 goto ok;
4872
4873 if (NAPI_GRO_CB(skb)->flush)
4874 goto normal;
4875
4876 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4877 struct sk_buff *nskb = napi->gro_list;
4878
4879 /* locate the end of the list to select the 'oldest' flow */
4880 while (nskb->next) {
4881 pp = &nskb->next;
4882 nskb = *pp;
4883 }
4884 *pp = NULL;
4885 nskb->next = NULL;
4886 napi_gro_complete(nskb);
4887 } else {
4888 napi->gro_count++;
4889 }
4890 NAPI_GRO_CB(skb)->count = 1;
4891 NAPI_GRO_CB(skb)->age = jiffies;
4892 NAPI_GRO_CB(skb)->last = skb;
4893 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4894 skb->next = napi->gro_list;
4895 napi->gro_list = skb;
4896 ret = GRO_HELD;
4897
4898 pull:
4899 grow = skb_gro_offset(skb) - skb_headlen(skb);
4900 if (grow > 0)
4901 gro_pull_from_frag0(skb, grow);
4902 ok:
4903 return ret;
4904
4905 normal:
4906 ret = GRO_NORMAL;
4907 goto pull;
4908 }
4909
gro_find_receive_by_type(__be16 type)4910 struct packet_offload *gro_find_receive_by_type(__be16 type)
4911 {
4912 struct list_head *offload_head = &offload_base;
4913 struct packet_offload *ptype;
4914
4915 list_for_each_entry_rcu(ptype, offload_head, list) {
4916 if (ptype->type != type || !ptype->callbacks.gro_receive)
4917 continue;
4918 return ptype;
4919 }
4920 return NULL;
4921 }
4922 EXPORT_SYMBOL(gro_find_receive_by_type);
4923
gro_find_complete_by_type(__be16 type)4924 struct packet_offload *gro_find_complete_by_type(__be16 type)
4925 {
4926 struct list_head *offload_head = &offload_base;
4927 struct packet_offload *ptype;
4928
4929 list_for_each_entry_rcu(ptype, offload_head, list) {
4930 if (ptype->type != type || !ptype->callbacks.gro_complete)
4931 continue;
4932 return ptype;
4933 }
4934 return NULL;
4935 }
4936 EXPORT_SYMBOL(gro_find_complete_by_type);
4937
napi_skb_free_stolen_head(struct sk_buff * skb)4938 static void napi_skb_free_stolen_head(struct sk_buff *skb)
4939 {
4940 skb_dst_drop(skb);
4941 secpath_reset(skb);
4942 kmem_cache_free(skbuff_head_cache, skb);
4943 }
4944
napi_skb_finish(gro_result_t ret,struct sk_buff * skb)4945 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4946 {
4947 switch (ret) {
4948 case GRO_NORMAL:
4949 if (netif_receive_skb_internal(skb))
4950 ret = GRO_DROP;
4951 break;
4952
4953 case GRO_DROP:
4954 kfree_skb(skb);
4955 break;
4956
4957 case GRO_MERGED_FREE:
4958 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4959 napi_skb_free_stolen_head(skb);
4960 else
4961 __kfree_skb(skb);
4962 break;
4963
4964 case GRO_HELD:
4965 case GRO_MERGED:
4966 case GRO_CONSUMED:
4967 break;
4968 }
4969
4970 return ret;
4971 }
4972
napi_gro_receive(struct napi_struct * napi,struct sk_buff * skb)4973 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4974 {
4975 skb_mark_napi_id(skb, napi);
4976 trace_napi_gro_receive_entry(skb);
4977
4978 skb_gro_reset_offset(skb);
4979
4980 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4981 }
4982 EXPORT_SYMBOL(napi_gro_receive);
4983
napi_reuse_skb(struct napi_struct * napi,struct sk_buff * skb)4984 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4985 {
4986 if (unlikely(skb->pfmemalloc)) {
4987 consume_skb(skb);
4988 return;
4989 }
4990 __skb_pull(skb, skb_headlen(skb));
4991 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4992 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4993 skb->vlan_tci = 0;
4994 skb->dev = napi->dev;
4995 skb->skb_iif = 0;
4996
4997 /* eth_type_trans() assumes pkt_type is PACKET_HOST */
4998 skb->pkt_type = PACKET_HOST;
4999
5000 skb->encapsulation = 0;
5001 skb_shinfo(skb)->gso_type = 0;
5002 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5003 secpath_reset(skb);
5004
5005 napi->skb = skb;
5006 }
5007
napi_get_frags(struct napi_struct * napi)5008 struct sk_buff *napi_get_frags(struct napi_struct *napi)
5009 {
5010 struct sk_buff *skb = napi->skb;
5011
5012 if (!skb) {
5013 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
5014 if (skb) {
5015 napi->skb = skb;
5016 skb_mark_napi_id(skb, napi);
5017 }
5018 }
5019 return skb;
5020 }
5021 EXPORT_SYMBOL(napi_get_frags);
5022
napi_frags_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)5023 static gro_result_t napi_frags_finish(struct napi_struct *napi,
5024 struct sk_buff *skb,
5025 gro_result_t ret)
5026 {
5027 switch (ret) {
5028 case GRO_NORMAL:
5029 case GRO_HELD:
5030 __skb_push(skb, ETH_HLEN);
5031 skb->protocol = eth_type_trans(skb, skb->dev);
5032 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
5033 ret = GRO_DROP;
5034 break;
5035
5036 case GRO_DROP:
5037 napi_reuse_skb(napi, skb);
5038 break;
5039
5040 case GRO_MERGED_FREE:
5041 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5042 napi_skb_free_stolen_head(skb);
5043 else
5044 napi_reuse_skb(napi, skb);
5045 break;
5046
5047 case GRO_MERGED:
5048 case GRO_CONSUMED:
5049 break;
5050 }
5051
5052 return ret;
5053 }
5054
5055 /* Upper GRO stack assumes network header starts at gro_offset=0
5056 * Drivers could call both napi_gro_frags() and napi_gro_receive()
5057 * We copy ethernet header into skb->data to have a common layout.
5058 */
napi_frags_skb(struct napi_struct * napi)5059 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
5060 {
5061 struct sk_buff *skb = napi->skb;
5062 const struct ethhdr *eth;
5063 unsigned int hlen = sizeof(*eth);
5064
5065 napi->skb = NULL;
5066
5067 skb_reset_mac_header(skb);
5068 skb_gro_reset_offset(skb);
5069
5070 if (unlikely(skb_gro_header_hard(skb, hlen))) {
5071 eth = skb_gro_header_slow(skb, hlen, 0);
5072 if (unlikely(!eth)) {
5073 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5074 __func__, napi->dev->name);
5075 napi_reuse_skb(napi, skb);
5076 return NULL;
5077 }
5078 } else {
5079 eth = (const struct ethhdr *)skb->data;
5080 gro_pull_from_frag0(skb, hlen);
5081 NAPI_GRO_CB(skb)->frag0 += hlen;
5082 NAPI_GRO_CB(skb)->frag0_len -= hlen;
5083 }
5084 __skb_pull(skb, hlen);
5085
5086 /*
5087 * This works because the only protocols we care about don't require
5088 * special handling.
5089 * We'll fix it up properly in napi_frags_finish()
5090 */
5091 skb->protocol = eth->h_proto;
5092
5093 return skb;
5094 }
5095
napi_gro_frags(struct napi_struct * napi)5096 gro_result_t napi_gro_frags(struct napi_struct *napi)
5097 {
5098 struct sk_buff *skb = napi_frags_skb(napi);
5099
5100 if (!skb)
5101 return GRO_DROP;
5102
5103 trace_napi_gro_frags_entry(skb);
5104
5105 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5106 }
5107 EXPORT_SYMBOL(napi_gro_frags);
5108
5109 /* Compute the checksum from gro_offset and return the folded value
5110 * after adding in any pseudo checksum.
5111 */
__skb_gro_checksum_complete(struct sk_buff * skb)5112 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5113 {
5114 __wsum wsum;
5115 __sum16 sum;
5116
5117 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5118
5119 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5120 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5121 if (likely(!sum)) {
5122 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5123 !skb->csum_complete_sw)
5124 netdev_rx_csum_fault(skb->dev);
5125 }
5126
5127 NAPI_GRO_CB(skb)->csum = wsum;
5128 NAPI_GRO_CB(skb)->csum_valid = 1;
5129
5130 return sum;
5131 }
5132 EXPORT_SYMBOL(__skb_gro_checksum_complete);
5133
net_rps_send_ipi(struct softnet_data * remsd)5134 static void net_rps_send_ipi(struct softnet_data *remsd)
5135 {
5136 #ifdef CONFIG_RPS
5137 while (remsd) {
5138 struct softnet_data *next = remsd->rps_ipi_next;
5139
5140 if (cpu_online(remsd->cpu))
5141 smp_call_function_single_async(remsd->cpu, &remsd->csd);
5142 remsd = next;
5143 }
5144 #endif
5145 }
5146
5147 /*
5148 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5149 * Note: called with local irq disabled, but exits with local irq enabled.
5150 */
net_rps_action_and_irq_enable(struct softnet_data * sd)5151 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5152 {
5153 #ifdef CONFIG_RPS
5154 struct softnet_data *remsd = sd->rps_ipi_list;
5155
5156 if (remsd) {
5157 sd->rps_ipi_list = NULL;
5158
5159 local_irq_enable();
5160
5161 /* Send pending IPI's to kick RPS processing on remote cpus. */
5162 net_rps_send_ipi(remsd);
5163 } else
5164 #endif
5165 local_irq_enable();
5166 }
5167
sd_has_rps_ipi_waiting(struct softnet_data * sd)5168 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5169 {
5170 #ifdef CONFIG_RPS
5171 return sd->rps_ipi_list != NULL;
5172 #else
5173 return false;
5174 #endif
5175 }
5176
process_backlog(struct napi_struct * napi,int quota)5177 static int process_backlog(struct napi_struct *napi, int quota)
5178 {
5179 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5180 bool again = true;
5181 int work = 0;
5182
5183 /* Check if we have pending ipi, its better to send them now,
5184 * not waiting net_rx_action() end.
5185 */
5186 if (sd_has_rps_ipi_waiting(sd)) {
5187 local_irq_disable();
5188 net_rps_action_and_irq_enable(sd);
5189 }
5190
5191 napi->weight = dev_rx_weight;
5192 while (again) {
5193 struct sk_buff *skb;
5194
5195 while ((skb = __skb_dequeue(&sd->process_queue))) {
5196 rcu_read_lock();
5197 __netif_receive_skb(skb);
5198 rcu_read_unlock();
5199 input_queue_head_incr(sd);
5200 if (++work >= quota)
5201 return work;
5202
5203 }
5204
5205 local_irq_disable();
5206 rps_lock(sd);
5207 if (skb_queue_empty(&sd->input_pkt_queue)) {
5208 /*
5209 * Inline a custom version of __napi_complete().
5210 * only current cpu owns and manipulates this napi,
5211 * and NAPI_STATE_SCHED is the only possible flag set
5212 * on backlog.
5213 * We can use a plain write instead of clear_bit(),
5214 * and we dont need an smp_mb() memory barrier.
5215 */
5216 napi->state = 0;
5217 again = false;
5218 } else {
5219 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5220 &sd->process_queue);
5221 }
5222 rps_unlock(sd);
5223 local_irq_enable();
5224 }
5225
5226 return work;
5227 }
5228
5229 /**
5230 * __napi_schedule - schedule for receive
5231 * @n: entry to schedule
5232 *
5233 * The entry's receive function will be scheduled to run.
5234 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5235 */
__napi_schedule(struct napi_struct * n)5236 void __napi_schedule(struct napi_struct *n)
5237 {
5238 unsigned long flags;
5239
5240 local_irq_save(flags);
5241 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5242 local_irq_restore(flags);
5243 }
5244 EXPORT_SYMBOL(__napi_schedule);
5245
5246 /**
5247 * napi_schedule_prep - check if napi can be scheduled
5248 * @n: napi context
5249 *
5250 * Test if NAPI routine is already running, and if not mark
5251 * it as running. This is used as a condition variable
5252 * insure only one NAPI poll instance runs. We also make
5253 * sure there is no pending NAPI disable.
5254 */
napi_schedule_prep(struct napi_struct * n)5255 bool napi_schedule_prep(struct napi_struct *n)
5256 {
5257 unsigned long val, new;
5258
5259 do {
5260 val = READ_ONCE(n->state);
5261 if (unlikely(val & NAPIF_STATE_DISABLE))
5262 return false;
5263 new = val | NAPIF_STATE_SCHED;
5264
5265 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5266 * This was suggested by Alexander Duyck, as compiler
5267 * emits better code than :
5268 * if (val & NAPIF_STATE_SCHED)
5269 * new |= NAPIF_STATE_MISSED;
5270 */
5271 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5272 NAPIF_STATE_MISSED;
5273 } while (cmpxchg(&n->state, val, new) != val);
5274
5275 return !(val & NAPIF_STATE_SCHED);
5276 }
5277 EXPORT_SYMBOL(napi_schedule_prep);
5278
5279 /**
5280 * __napi_schedule_irqoff - schedule for receive
5281 * @n: entry to schedule
5282 *
5283 * Variant of __napi_schedule() assuming hard irqs are masked
5284 */
__napi_schedule_irqoff(struct napi_struct * n)5285 void __napi_schedule_irqoff(struct napi_struct *n)
5286 {
5287 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5288 }
5289 EXPORT_SYMBOL(__napi_schedule_irqoff);
5290
napi_complete_done(struct napi_struct * n,int work_done)5291 bool napi_complete_done(struct napi_struct *n, int work_done)
5292 {
5293 unsigned long flags, val, new;
5294
5295 /*
5296 * 1) Don't let napi dequeue from the cpu poll list
5297 * just in case its running on a different cpu.
5298 * 2) If we are busy polling, do nothing here, we have
5299 * the guarantee we will be called later.
5300 */
5301 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5302 NAPIF_STATE_IN_BUSY_POLL)))
5303 return false;
5304
5305 if (n->gro_list) {
5306 unsigned long timeout = 0;
5307
5308 if (work_done)
5309 timeout = n->dev->gro_flush_timeout;
5310
5311 /* When the NAPI instance uses a timeout and keeps postponing
5312 * it, we need to bound somehow the time packets are kept in
5313 * the GRO layer
5314 */
5315 napi_gro_flush(n, !!timeout);
5316 if (timeout)
5317 hrtimer_start(&n->timer, ns_to_ktime(timeout),
5318 HRTIMER_MODE_REL_PINNED);
5319 }
5320 if (unlikely(!list_empty(&n->poll_list))) {
5321 /* If n->poll_list is not empty, we need to mask irqs */
5322 local_irq_save(flags);
5323 list_del_init(&n->poll_list);
5324 local_irq_restore(flags);
5325 }
5326
5327 do {
5328 val = READ_ONCE(n->state);
5329
5330 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5331
5332 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5333
5334 /* If STATE_MISSED was set, leave STATE_SCHED set,
5335 * because we will call napi->poll() one more time.
5336 * This C code was suggested by Alexander Duyck to help gcc.
5337 */
5338 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5339 NAPIF_STATE_SCHED;
5340 } while (cmpxchg(&n->state, val, new) != val);
5341
5342 if (unlikely(val & NAPIF_STATE_MISSED)) {
5343 __napi_schedule(n);
5344 return false;
5345 }
5346
5347 return true;
5348 }
5349 EXPORT_SYMBOL(napi_complete_done);
5350
5351 /* must be called under rcu_read_lock(), as we dont take a reference */
napi_by_id(unsigned int napi_id)5352 static struct napi_struct *napi_by_id(unsigned int napi_id)
5353 {
5354 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5355 struct napi_struct *napi;
5356
5357 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5358 if (napi->napi_id == napi_id)
5359 return napi;
5360
5361 return NULL;
5362 }
5363
5364 #if defined(CONFIG_NET_RX_BUSY_POLL)
5365
5366 #define BUSY_POLL_BUDGET 8
5367
busy_poll_stop(struct napi_struct * napi,void * have_poll_lock)5368 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5369 {
5370 int rc;
5371
5372 /* Busy polling means there is a high chance device driver hard irq
5373 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
5374 * set in napi_schedule_prep().
5375 * Since we are about to call napi->poll() once more, we can safely
5376 * clear NAPI_STATE_MISSED.
5377 *
5378 * Note: x86 could use a single "lock and ..." instruction
5379 * to perform these two clear_bit()
5380 */
5381 clear_bit(NAPI_STATE_MISSED, &napi->state);
5382 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5383
5384 local_bh_disable();
5385
5386 /* All we really want here is to re-enable device interrupts.
5387 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
5388 */
5389 rc = napi->poll(napi, BUSY_POLL_BUDGET);
5390 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5391 netpoll_poll_unlock(have_poll_lock);
5392 if (rc == BUSY_POLL_BUDGET)
5393 __napi_schedule(napi);
5394 local_bh_enable();
5395 }
5396
napi_busy_loop(unsigned int napi_id,bool (* loop_end)(void *,unsigned long),void * loop_end_arg)5397 void napi_busy_loop(unsigned int napi_id,
5398 bool (*loop_end)(void *, unsigned long),
5399 void *loop_end_arg)
5400 {
5401 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
5402 int (*napi_poll)(struct napi_struct *napi, int budget);
5403 void *have_poll_lock = NULL;
5404 struct napi_struct *napi;
5405
5406 restart:
5407 napi_poll = NULL;
5408
5409 rcu_read_lock();
5410
5411 napi = napi_by_id(napi_id);
5412 if (!napi)
5413 goto out;
5414
5415 preempt_disable();
5416 for (;;) {
5417 int work = 0;
5418
5419 local_bh_disable();
5420 if (!napi_poll) {
5421 unsigned long val = READ_ONCE(napi->state);
5422
5423 /* If multiple threads are competing for this napi,
5424 * we avoid dirtying napi->state as much as we can.
5425 */
5426 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5427 NAPIF_STATE_IN_BUSY_POLL))
5428 goto count;
5429 if (cmpxchg(&napi->state, val,
5430 val | NAPIF_STATE_IN_BUSY_POLL |
5431 NAPIF_STATE_SCHED) != val)
5432 goto count;
5433 have_poll_lock = netpoll_poll_lock(napi);
5434 napi_poll = napi->poll;
5435 }
5436 work = napi_poll(napi, BUSY_POLL_BUDGET);
5437 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
5438 count:
5439 if (work > 0)
5440 __NET_ADD_STATS(dev_net(napi->dev),
5441 LINUX_MIB_BUSYPOLLRXPACKETS, work);
5442 local_bh_enable();
5443
5444 if (!loop_end || loop_end(loop_end_arg, start_time))
5445 break;
5446
5447 if (unlikely(need_resched())) {
5448 if (napi_poll)
5449 busy_poll_stop(napi, have_poll_lock);
5450 preempt_enable();
5451 rcu_read_unlock();
5452 cond_resched();
5453 if (loop_end(loop_end_arg, start_time))
5454 return;
5455 goto restart;
5456 }
5457 cpu_relax();
5458 }
5459 if (napi_poll)
5460 busy_poll_stop(napi, have_poll_lock);
5461 preempt_enable();
5462 out:
5463 rcu_read_unlock();
5464 }
5465 EXPORT_SYMBOL(napi_busy_loop);
5466
5467 #endif /* CONFIG_NET_RX_BUSY_POLL */
5468
napi_hash_add(struct napi_struct * napi)5469 static void napi_hash_add(struct napi_struct *napi)
5470 {
5471 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5472 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5473 return;
5474
5475 spin_lock(&napi_hash_lock);
5476
5477 /* 0..NR_CPUS range is reserved for sender_cpu use */
5478 do {
5479 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
5480 napi_gen_id = MIN_NAPI_ID;
5481 } while (napi_by_id(napi_gen_id));
5482 napi->napi_id = napi_gen_id;
5483
5484 hlist_add_head_rcu(&napi->napi_hash_node,
5485 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5486
5487 spin_unlock(&napi_hash_lock);
5488 }
5489
5490 /* Warning : caller is responsible to make sure rcu grace period
5491 * is respected before freeing memory containing @napi
5492 */
napi_hash_del(struct napi_struct * napi)5493 bool napi_hash_del(struct napi_struct *napi)
5494 {
5495 bool rcu_sync_needed = false;
5496
5497 spin_lock(&napi_hash_lock);
5498
5499 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5500 rcu_sync_needed = true;
5501 hlist_del_rcu(&napi->napi_hash_node);
5502 }
5503 spin_unlock(&napi_hash_lock);
5504 return rcu_sync_needed;
5505 }
5506 EXPORT_SYMBOL_GPL(napi_hash_del);
5507
napi_watchdog(struct hrtimer * timer)5508 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5509 {
5510 struct napi_struct *napi;
5511
5512 napi = container_of(timer, struct napi_struct, timer);
5513
5514 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
5515 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
5516 */
5517 if (napi->gro_list && !napi_disable_pending(napi) &&
5518 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5519 __napi_schedule_irqoff(napi);
5520
5521 return HRTIMER_NORESTART;
5522 }
5523
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)5524 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5525 int (*poll)(struct napi_struct *, int), int weight)
5526 {
5527 INIT_LIST_HEAD(&napi->poll_list);
5528 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5529 napi->timer.function = napi_watchdog;
5530 napi->gro_count = 0;
5531 napi->gro_list = NULL;
5532 napi->skb = NULL;
5533 napi->poll = poll;
5534 if (weight > NAPI_POLL_WEIGHT)
5535 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5536 weight, dev->name);
5537 napi->weight = weight;
5538 list_add(&napi->dev_list, &dev->napi_list);
5539 napi->dev = dev;
5540 #ifdef CONFIG_NETPOLL
5541 napi->poll_owner = -1;
5542 #endif
5543 set_bit(NAPI_STATE_SCHED, &napi->state);
5544 napi_hash_add(napi);
5545 }
5546 EXPORT_SYMBOL(netif_napi_add);
5547
napi_disable(struct napi_struct * n)5548 void napi_disable(struct napi_struct *n)
5549 {
5550 might_sleep();
5551 set_bit(NAPI_STATE_DISABLE, &n->state);
5552
5553 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5554 msleep(1);
5555 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5556 msleep(1);
5557
5558 hrtimer_cancel(&n->timer);
5559
5560 clear_bit(NAPI_STATE_DISABLE, &n->state);
5561 }
5562 EXPORT_SYMBOL(napi_disable);
5563
5564 /* Must be called in process context */
netif_napi_del(struct napi_struct * napi)5565 void netif_napi_del(struct napi_struct *napi)
5566 {
5567 might_sleep();
5568 if (napi_hash_del(napi))
5569 synchronize_net();
5570 list_del_init(&napi->dev_list);
5571 napi_free_frags(napi);
5572
5573 kfree_skb_list(napi->gro_list);
5574 napi->gro_list = NULL;
5575 napi->gro_count = 0;
5576 }
5577 EXPORT_SYMBOL(netif_napi_del);
5578
napi_poll(struct napi_struct * n,struct list_head * repoll)5579 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5580 {
5581 void *have;
5582 int work, weight;
5583
5584 list_del_init(&n->poll_list);
5585
5586 have = netpoll_poll_lock(n);
5587
5588 weight = n->weight;
5589
5590 /* This NAPI_STATE_SCHED test is for avoiding a race
5591 * with netpoll's poll_napi(). Only the entity which
5592 * obtains the lock and sees NAPI_STATE_SCHED set will
5593 * actually make the ->poll() call. Therefore we avoid
5594 * accidentally calling ->poll() when NAPI is not scheduled.
5595 */
5596 work = 0;
5597 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5598 work = n->poll(n, weight);
5599 trace_napi_poll(n, work, weight);
5600 }
5601
5602 WARN_ON_ONCE(work > weight);
5603
5604 if (likely(work < weight))
5605 goto out_unlock;
5606
5607 /* Drivers must not modify the NAPI state if they
5608 * consume the entire weight. In such cases this code
5609 * still "owns" the NAPI instance and therefore can
5610 * move the instance around on the list at-will.
5611 */
5612 if (unlikely(napi_disable_pending(n))) {
5613 napi_complete(n);
5614 goto out_unlock;
5615 }
5616
5617 if (n->gro_list) {
5618 /* flush too old packets
5619 * If HZ < 1000, flush all packets.
5620 */
5621 napi_gro_flush(n, HZ >= 1000);
5622 }
5623
5624 /* Some drivers may have called napi_schedule
5625 * prior to exhausting their budget.
5626 */
5627 if (unlikely(!list_empty(&n->poll_list))) {
5628 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5629 n->dev ? n->dev->name : "backlog");
5630 goto out_unlock;
5631 }
5632
5633 list_add_tail(&n->poll_list, repoll);
5634
5635 out_unlock:
5636 netpoll_poll_unlock(have);
5637
5638 return work;
5639 }
5640
net_rx_action(struct softirq_action * h)5641 static __latent_entropy void net_rx_action(struct softirq_action *h)
5642 {
5643 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5644 unsigned long time_limit = jiffies +
5645 usecs_to_jiffies(netdev_budget_usecs);
5646 int budget = netdev_budget;
5647 LIST_HEAD(list);
5648 LIST_HEAD(repoll);
5649
5650 local_irq_disable();
5651 list_splice_init(&sd->poll_list, &list);
5652 local_irq_enable();
5653
5654 for (;;) {
5655 struct napi_struct *n;
5656
5657 if (list_empty(&list)) {
5658 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5659 goto out;
5660 break;
5661 }
5662
5663 n = list_first_entry(&list, struct napi_struct, poll_list);
5664 budget -= napi_poll(n, &repoll);
5665
5666 /* If softirq window is exhausted then punt.
5667 * Allow this to run for 2 jiffies since which will allow
5668 * an average latency of 1.5/HZ.
5669 */
5670 if (unlikely(budget <= 0 ||
5671 time_after_eq(jiffies, time_limit))) {
5672 sd->time_squeeze++;
5673 break;
5674 }
5675 }
5676
5677 local_irq_disable();
5678
5679 list_splice_tail_init(&sd->poll_list, &list);
5680 list_splice_tail(&repoll, &list);
5681 list_splice(&list, &sd->poll_list);
5682 if (!list_empty(&sd->poll_list))
5683 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5684
5685 net_rps_action_and_irq_enable(sd);
5686 out:
5687 __kfree_skb_flush();
5688 }
5689
5690 struct netdev_adjacent {
5691 struct net_device *dev;
5692
5693 /* upper master flag, there can only be one master device per list */
5694 bool master;
5695
5696 /* counter for the number of times this device was added to us */
5697 u16 ref_nr;
5698
5699 /* private field for the users */
5700 void *private;
5701
5702 struct list_head list;
5703 struct rcu_head rcu;
5704 };
5705
__netdev_find_adj(struct net_device * adj_dev,struct list_head * adj_list)5706 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5707 struct list_head *adj_list)
5708 {
5709 struct netdev_adjacent *adj;
5710
5711 list_for_each_entry(adj, adj_list, list) {
5712 if (adj->dev == adj_dev)
5713 return adj;
5714 }
5715 return NULL;
5716 }
5717
__netdev_has_upper_dev(struct net_device * upper_dev,void * data)5718 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5719 {
5720 struct net_device *dev = data;
5721
5722 return upper_dev == dev;
5723 }
5724
5725 /**
5726 * netdev_has_upper_dev - Check if device is linked to an upper device
5727 * @dev: device
5728 * @upper_dev: upper device to check
5729 *
5730 * Find out if a device is linked to specified upper device and return true
5731 * in case it is. Note that this checks only immediate upper device,
5732 * not through a complete stack of devices. The caller must hold the RTNL lock.
5733 */
netdev_has_upper_dev(struct net_device * dev,struct net_device * upper_dev)5734 bool netdev_has_upper_dev(struct net_device *dev,
5735 struct net_device *upper_dev)
5736 {
5737 ASSERT_RTNL();
5738
5739 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5740 upper_dev);
5741 }
5742 EXPORT_SYMBOL(netdev_has_upper_dev);
5743
5744 /**
5745 * netdev_has_upper_dev_all - Check if device is linked to an upper device
5746 * @dev: device
5747 * @upper_dev: upper device to check
5748 *
5749 * Find out if a device is linked to specified upper device and return true
5750 * in case it is. Note that this checks the entire upper device chain.
5751 * The caller must hold rcu lock.
5752 */
5753
netdev_has_upper_dev_all_rcu(struct net_device * dev,struct net_device * upper_dev)5754 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5755 struct net_device *upper_dev)
5756 {
5757 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5758 upper_dev);
5759 }
5760 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5761
5762 /**
5763 * netdev_has_any_upper_dev - Check if device is linked to some device
5764 * @dev: device
5765 *
5766 * Find out if a device is linked to an upper device and return true in case
5767 * it is. The caller must hold the RTNL lock.
5768 */
netdev_has_any_upper_dev(struct net_device * dev)5769 bool netdev_has_any_upper_dev(struct net_device *dev)
5770 {
5771 ASSERT_RTNL();
5772
5773 return !list_empty(&dev->adj_list.upper);
5774 }
5775 EXPORT_SYMBOL(netdev_has_any_upper_dev);
5776
5777 /**
5778 * netdev_master_upper_dev_get - Get master upper device
5779 * @dev: device
5780 *
5781 * Find a master upper device and return pointer to it or NULL in case
5782 * it's not there. The caller must hold the RTNL lock.
5783 */
netdev_master_upper_dev_get(struct net_device * dev)5784 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5785 {
5786 struct netdev_adjacent *upper;
5787
5788 ASSERT_RTNL();
5789
5790 if (list_empty(&dev->adj_list.upper))
5791 return NULL;
5792
5793 upper = list_first_entry(&dev->adj_list.upper,
5794 struct netdev_adjacent, list);
5795 if (likely(upper->master))
5796 return upper->dev;
5797 return NULL;
5798 }
5799 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5800
5801 /**
5802 * netdev_has_any_lower_dev - Check if device is linked to some device
5803 * @dev: device
5804 *
5805 * Find out if a device is linked to a lower device and return true in case
5806 * it is. The caller must hold the RTNL lock.
5807 */
netdev_has_any_lower_dev(struct net_device * dev)5808 static bool netdev_has_any_lower_dev(struct net_device *dev)
5809 {
5810 ASSERT_RTNL();
5811
5812 return !list_empty(&dev->adj_list.lower);
5813 }
5814
netdev_adjacent_get_private(struct list_head * adj_list)5815 void *netdev_adjacent_get_private(struct list_head *adj_list)
5816 {
5817 struct netdev_adjacent *adj;
5818
5819 adj = list_entry(adj_list, struct netdev_adjacent, list);
5820
5821 return adj->private;
5822 }
5823 EXPORT_SYMBOL(netdev_adjacent_get_private);
5824
5825 /**
5826 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5827 * @dev: device
5828 * @iter: list_head ** of the current position
5829 *
5830 * Gets the next device from the dev's upper list, starting from iter
5831 * position. The caller must hold RCU read lock.
5832 */
netdev_upper_get_next_dev_rcu(struct net_device * dev,struct list_head ** iter)5833 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5834 struct list_head **iter)
5835 {
5836 struct netdev_adjacent *upper;
5837
5838 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5839
5840 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5841
5842 if (&upper->list == &dev->adj_list.upper)
5843 return NULL;
5844
5845 *iter = &upper->list;
5846
5847 return upper->dev;
5848 }
5849 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5850
netdev_next_upper_dev_rcu(struct net_device * dev,struct list_head ** iter)5851 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5852 struct list_head **iter)
5853 {
5854 struct netdev_adjacent *upper;
5855
5856 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5857
5858 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5859
5860 if (&upper->list == &dev->adj_list.upper)
5861 return NULL;
5862
5863 *iter = &upper->list;
5864
5865 return upper->dev;
5866 }
5867
netdev_walk_all_upper_dev_rcu(struct net_device * dev,int (* fn)(struct net_device * dev,void * data),void * data)5868 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5869 int (*fn)(struct net_device *dev,
5870 void *data),
5871 void *data)
5872 {
5873 struct net_device *udev;
5874 struct list_head *iter;
5875 int ret;
5876
5877 for (iter = &dev->adj_list.upper,
5878 udev = netdev_next_upper_dev_rcu(dev, &iter);
5879 udev;
5880 udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5881 /* first is the upper device itself */
5882 ret = fn(udev, data);
5883 if (ret)
5884 return ret;
5885
5886 /* then look at all of its upper devices */
5887 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5888 if (ret)
5889 return ret;
5890 }
5891
5892 return 0;
5893 }
5894 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5895
5896 /**
5897 * netdev_lower_get_next_private - Get the next ->private from the
5898 * lower neighbour list
5899 * @dev: device
5900 * @iter: list_head ** of the current position
5901 *
5902 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5903 * list, starting from iter position. The caller must hold either hold the
5904 * RTNL lock or its own locking that guarantees that the neighbour lower
5905 * list will remain unchanged.
5906 */
netdev_lower_get_next_private(struct net_device * dev,struct list_head ** iter)5907 void *netdev_lower_get_next_private(struct net_device *dev,
5908 struct list_head **iter)
5909 {
5910 struct netdev_adjacent *lower;
5911
5912 lower = list_entry(*iter, struct netdev_adjacent, list);
5913
5914 if (&lower->list == &dev->adj_list.lower)
5915 return NULL;
5916
5917 *iter = lower->list.next;
5918
5919 return lower->private;
5920 }
5921 EXPORT_SYMBOL(netdev_lower_get_next_private);
5922
5923 /**
5924 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5925 * lower neighbour list, RCU
5926 * variant
5927 * @dev: device
5928 * @iter: list_head ** of the current position
5929 *
5930 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5931 * list, starting from iter position. The caller must hold RCU read lock.
5932 */
netdev_lower_get_next_private_rcu(struct net_device * dev,struct list_head ** iter)5933 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5934 struct list_head **iter)
5935 {
5936 struct netdev_adjacent *lower;
5937
5938 WARN_ON_ONCE(!rcu_read_lock_held());
5939
5940 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5941
5942 if (&lower->list == &dev->adj_list.lower)
5943 return NULL;
5944
5945 *iter = &lower->list;
5946
5947 return lower->private;
5948 }
5949 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5950
5951 /**
5952 * netdev_lower_get_next - Get the next device from the lower neighbour
5953 * list
5954 * @dev: device
5955 * @iter: list_head ** of the current position
5956 *
5957 * Gets the next netdev_adjacent from the dev's lower neighbour
5958 * list, starting from iter position. The caller must hold RTNL lock or
5959 * its own locking that guarantees that the neighbour lower
5960 * list will remain unchanged.
5961 */
netdev_lower_get_next(struct net_device * dev,struct list_head ** iter)5962 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5963 {
5964 struct netdev_adjacent *lower;
5965
5966 lower = list_entry(*iter, struct netdev_adjacent, list);
5967
5968 if (&lower->list == &dev->adj_list.lower)
5969 return NULL;
5970
5971 *iter = lower->list.next;
5972
5973 return lower->dev;
5974 }
5975 EXPORT_SYMBOL(netdev_lower_get_next);
5976
netdev_next_lower_dev(struct net_device * dev,struct list_head ** iter)5977 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5978 struct list_head **iter)
5979 {
5980 struct netdev_adjacent *lower;
5981
5982 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5983
5984 if (&lower->list == &dev->adj_list.lower)
5985 return NULL;
5986
5987 *iter = &lower->list;
5988
5989 return lower->dev;
5990 }
5991
netdev_walk_all_lower_dev(struct net_device * dev,int (* fn)(struct net_device * dev,void * data),void * data)5992 int netdev_walk_all_lower_dev(struct net_device *dev,
5993 int (*fn)(struct net_device *dev,
5994 void *data),
5995 void *data)
5996 {
5997 struct net_device *ldev;
5998 struct list_head *iter;
5999 int ret;
6000
6001 for (iter = &dev->adj_list.lower,
6002 ldev = netdev_next_lower_dev(dev, &iter);
6003 ldev;
6004 ldev = netdev_next_lower_dev(dev, &iter)) {
6005 /* first is the lower device itself */
6006 ret = fn(ldev, data);
6007 if (ret)
6008 return ret;
6009
6010 /* then look at all of its lower devices */
6011 ret = netdev_walk_all_lower_dev(ldev, fn, data);
6012 if (ret)
6013 return ret;
6014 }
6015
6016 return 0;
6017 }
6018 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
6019
netdev_next_lower_dev_rcu(struct net_device * dev,struct list_head ** iter)6020 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
6021 struct list_head **iter)
6022 {
6023 struct netdev_adjacent *lower;
6024
6025 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6026 if (&lower->list == &dev->adj_list.lower)
6027 return NULL;
6028
6029 *iter = &lower->list;
6030
6031 return lower->dev;
6032 }
6033
netdev_walk_all_lower_dev_rcu(struct net_device * dev,int (* fn)(struct net_device * dev,void * data),void * data)6034 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
6035 int (*fn)(struct net_device *dev,
6036 void *data),
6037 void *data)
6038 {
6039 struct net_device *ldev;
6040 struct list_head *iter;
6041 int ret;
6042
6043 for (iter = &dev->adj_list.lower,
6044 ldev = netdev_next_lower_dev_rcu(dev, &iter);
6045 ldev;
6046 ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
6047 /* first is the lower device itself */
6048 ret = fn(ldev, data);
6049 if (ret)
6050 return ret;
6051
6052 /* then look at all of its lower devices */
6053 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
6054 if (ret)
6055 return ret;
6056 }
6057
6058 return 0;
6059 }
6060 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
6061
6062 /**
6063 * netdev_lower_get_first_private_rcu - Get the first ->private from the
6064 * lower neighbour list, RCU
6065 * variant
6066 * @dev: device
6067 *
6068 * Gets the first netdev_adjacent->private from the dev's lower neighbour
6069 * list. The caller must hold RCU read lock.
6070 */
netdev_lower_get_first_private_rcu(struct net_device * dev)6071 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
6072 {
6073 struct netdev_adjacent *lower;
6074
6075 lower = list_first_or_null_rcu(&dev->adj_list.lower,
6076 struct netdev_adjacent, list);
6077 if (lower)
6078 return lower->private;
6079 return NULL;
6080 }
6081 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
6082
6083 /**
6084 * netdev_master_upper_dev_get_rcu - Get master upper device
6085 * @dev: device
6086 *
6087 * Find a master upper device and return pointer to it or NULL in case
6088 * it's not there. The caller must hold the RCU read lock.
6089 */
netdev_master_upper_dev_get_rcu(struct net_device * dev)6090 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
6091 {
6092 struct netdev_adjacent *upper;
6093
6094 upper = list_first_or_null_rcu(&dev->adj_list.upper,
6095 struct netdev_adjacent, list);
6096 if (upper && likely(upper->master))
6097 return upper->dev;
6098 return NULL;
6099 }
6100 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6101
netdev_adjacent_sysfs_add(struct net_device * dev,struct net_device * adj_dev,struct list_head * dev_list)6102 static int netdev_adjacent_sysfs_add(struct net_device *dev,
6103 struct net_device *adj_dev,
6104 struct list_head *dev_list)
6105 {
6106 char linkname[IFNAMSIZ+7];
6107
6108 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6109 "upper_%s" : "lower_%s", adj_dev->name);
6110 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6111 linkname);
6112 }
netdev_adjacent_sysfs_del(struct net_device * dev,char * name,struct list_head * dev_list)6113 static void netdev_adjacent_sysfs_del(struct net_device *dev,
6114 char *name,
6115 struct list_head *dev_list)
6116 {
6117 char linkname[IFNAMSIZ+7];
6118
6119 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6120 "upper_%s" : "lower_%s", name);
6121 sysfs_remove_link(&(dev->dev.kobj), linkname);
6122 }
6123
netdev_adjacent_is_neigh_list(struct net_device * dev,struct net_device * adj_dev,struct list_head * dev_list)6124 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6125 struct net_device *adj_dev,
6126 struct list_head *dev_list)
6127 {
6128 return (dev_list == &dev->adj_list.upper ||
6129 dev_list == &dev->adj_list.lower) &&
6130 net_eq(dev_net(dev), dev_net(adj_dev));
6131 }
6132
__netdev_adjacent_dev_insert(struct net_device * dev,struct net_device * adj_dev,struct list_head * dev_list,void * private,bool master)6133 static int __netdev_adjacent_dev_insert(struct net_device *dev,
6134 struct net_device *adj_dev,
6135 struct list_head *dev_list,
6136 void *private, bool master)
6137 {
6138 struct netdev_adjacent *adj;
6139 int ret;
6140
6141 adj = __netdev_find_adj(adj_dev, dev_list);
6142
6143 if (adj) {
6144 adj->ref_nr += 1;
6145 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6146 dev->name, adj_dev->name, adj->ref_nr);
6147
6148 return 0;
6149 }
6150
6151 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6152 if (!adj)
6153 return -ENOMEM;
6154
6155 adj->dev = adj_dev;
6156 adj->master = master;
6157 adj->ref_nr = 1;
6158 adj->private = private;
6159 dev_hold(adj_dev);
6160
6161 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6162 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
6163
6164 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
6165 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
6166 if (ret)
6167 goto free_adj;
6168 }
6169
6170 /* Ensure that master link is always the first item in list. */
6171 if (master) {
6172 ret = sysfs_create_link(&(dev->dev.kobj),
6173 &(adj_dev->dev.kobj), "master");
6174 if (ret)
6175 goto remove_symlinks;
6176
6177 list_add_rcu(&adj->list, dev_list);
6178 } else {
6179 list_add_tail_rcu(&adj->list, dev_list);
6180 }
6181
6182 return 0;
6183
6184 remove_symlinks:
6185 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6186 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6187 free_adj:
6188 kfree(adj);
6189 dev_put(adj_dev);
6190
6191 return ret;
6192 }
6193
__netdev_adjacent_dev_remove(struct net_device * dev,struct net_device * adj_dev,u16 ref_nr,struct list_head * dev_list)6194 static void __netdev_adjacent_dev_remove(struct net_device *dev,
6195 struct net_device *adj_dev,
6196 u16 ref_nr,
6197 struct list_head *dev_list)
6198 {
6199 struct netdev_adjacent *adj;
6200
6201 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6202 dev->name, adj_dev->name, ref_nr);
6203
6204 adj = __netdev_find_adj(adj_dev, dev_list);
6205
6206 if (!adj) {
6207 pr_err("Adjacency does not exist for device %s from %s\n",
6208 dev->name, adj_dev->name);
6209 WARN_ON(1);
6210 return;
6211 }
6212
6213 if (adj->ref_nr > ref_nr) {
6214 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6215 dev->name, adj_dev->name, ref_nr,
6216 adj->ref_nr - ref_nr);
6217 adj->ref_nr -= ref_nr;
6218 return;
6219 }
6220
6221 if (adj->master)
6222 sysfs_remove_link(&(dev->dev.kobj), "master");
6223
6224 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6225 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6226
6227 list_del_rcu(&adj->list);
6228 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
6229 adj_dev->name, dev->name, adj_dev->name);
6230 dev_put(adj_dev);
6231 kfree_rcu(adj, rcu);
6232 }
6233
__netdev_adjacent_dev_link_lists(struct net_device * dev,struct net_device * upper_dev,struct list_head * up_list,struct list_head * down_list,void * private,bool master)6234 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6235 struct net_device *upper_dev,
6236 struct list_head *up_list,
6237 struct list_head *down_list,
6238 void *private, bool master)
6239 {
6240 int ret;
6241
6242 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
6243 private, master);
6244 if (ret)
6245 return ret;
6246
6247 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
6248 private, false);
6249 if (ret) {
6250 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
6251 return ret;
6252 }
6253
6254 return 0;
6255 }
6256
__netdev_adjacent_dev_unlink_lists(struct net_device * dev,struct net_device * upper_dev,u16 ref_nr,struct list_head * up_list,struct list_head * down_list)6257 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
6258 struct net_device *upper_dev,
6259 u16 ref_nr,
6260 struct list_head *up_list,
6261 struct list_head *down_list)
6262 {
6263 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
6264 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
6265 }
6266
__netdev_adjacent_dev_link_neighbour(struct net_device * dev,struct net_device * upper_dev,void * private,bool master)6267 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
6268 struct net_device *upper_dev,
6269 void *private, bool master)
6270 {
6271 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
6272 &dev->adj_list.upper,
6273 &upper_dev->adj_list.lower,
6274 private, master);
6275 }
6276
__netdev_adjacent_dev_unlink_neighbour(struct net_device * dev,struct net_device * upper_dev)6277 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
6278 struct net_device *upper_dev)
6279 {
6280 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
6281 &dev->adj_list.upper,
6282 &upper_dev->adj_list.lower);
6283 }
6284
__netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev,bool master,void * upper_priv,void * upper_info)6285 static int __netdev_upper_dev_link(struct net_device *dev,
6286 struct net_device *upper_dev, bool master,
6287 void *upper_priv, void *upper_info)
6288 {
6289 struct netdev_notifier_changeupper_info changeupper_info;
6290 int ret = 0;
6291
6292 ASSERT_RTNL();
6293
6294 if (dev == upper_dev)
6295 return -EBUSY;
6296
6297 /* To prevent loops, check if dev is not upper device to upper_dev. */
6298 if (netdev_has_upper_dev(upper_dev, dev))
6299 return -EBUSY;
6300
6301 if (netdev_has_upper_dev(dev, upper_dev))
6302 return -EEXIST;
6303
6304 if (master && netdev_master_upper_dev_get(dev))
6305 return -EBUSY;
6306
6307 changeupper_info.upper_dev = upper_dev;
6308 changeupper_info.master = master;
6309 changeupper_info.linking = true;
6310 changeupper_info.upper_info = upper_info;
6311
6312 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6313 &changeupper_info.info);
6314 ret = notifier_to_errno(ret);
6315 if (ret)
6316 return ret;
6317
6318 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
6319 master);
6320 if (ret)
6321 return ret;
6322
6323 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6324 &changeupper_info.info);
6325 ret = notifier_to_errno(ret);
6326 if (ret)
6327 goto rollback;
6328
6329 return 0;
6330
6331 rollback:
6332 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6333
6334 return ret;
6335 }
6336
6337 /**
6338 * netdev_upper_dev_link - Add a link to the upper device
6339 * @dev: device
6340 * @upper_dev: new upper device
6341 *
6342 * Adds a link to device which is upper to this one. The caller must hold
6343 * the RTNL lock. On a failure a negative errno code is returned.
6344 * On success the reference counts are adjusted and the function
6345 * returns zero.
6346 */
netdev_upper_dev_link(struct net_device * dev,struct net_device * upper_dev)6347 int netdev_upper_dev_link(struct net_device *dev,
6348 struct net_device *upper_dev)
6349 {
6350 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
6351 }
6352 EXPORT_SYMBOL(netdev_upper_dev_link);
6353
6354 /**
6355 * netdev_master_upper_dev_link - Add a master link to the upper device
6356 * @dev: device
6357 * @upper_dev: new upper device
6358 * @upper_priv: upper device private
6359 * @upper_info: upper info to be passed down via notifier
6360 *
6361 * Adds a link to device which is upper to this one. In this case, only
6362 * one master upper device can be linked, although other non-master devices
6363 * might be linked as well. The caller must hold the RTNL lock.
6364 * On a failure a negative errno code is returned. On success the reference
6365 * counts are adjusted and the function returns zero.
6366 */
netdev_master_upper_dev_link(struct net_device * dev,struct net_device * upper_dev,void * upper_priv,void * upper_info)6367 int netdev_master_upper_dev_link(struct net_device *dev,
6368 struct net_device *upper_dev,
6369 void *upper_priv, void *upper_info)
6370 {
6371 return __netdev_upper_dev_link(dev, upper_dev, true,
6372 upper_priv, upper_info);
6373 }
6374 EXPORT_SYMBOL(netdev_master_upper_dev_link);
6375
6376 /**
6377 * netdev_upper_dev_unlink - Removes a link to upper device
6378 * @dev: device
6379 * @upper_dev: new upper device
6380 *
6381 * Removes a link to device which is upper to this one. The caller must hold
6382 * the RTNL lock.
6383 */
netdev_upper_dev_unlink(struct net_device * dev,struct net_device * upper_dev)6384 void netdev_upper_dev_unlink(struct net_device *dev,
6385 struct net_device *upper_dev)
6386 {
6387 struct netdev_notifier_changeupper_info changeupper_info;
6388
6389 ASSERT_RTNL();
6390
6391 changeupper_info.upper_dev = upper_dev;
6392 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
6393 changeupper_info.linking = false;
6394
6395 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6396 &changeupper_info.info);
6397
6398 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6399
6400 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6401 &changeupper_info.info);
6402 }
6403 EXPORT_SYMBOL(netdev_upper_dev_unlink);
6404
6405 /**
6406 * netdev_bonding_info_change - Dispatch event about slave change
6407 * @dev: device
6408 * @bonding_info: info to dispatch
6409 *
6410 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
6411 * The caller must hold the RTNL lock.
6412 */
netdev_bonding_info_change(struct net_device * dev,struct netdev_bonding_info * bonding_info)6413 void netdev_bonding_info_change(struct net_device *dev,
6414 struct netdev_bonding_info *bonding_info)
6415 {
6416 struct netdev_notifier_bonding_info info;
6417
6418 memcpy(&info.bonding_info, bonding_info,
6419 sizeof(struct netdev_bonding_info));
6420 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
6421 &info.info);
6422 }
6423 EXPORT_SYMBOL(netdev_bonding_info_change);
6424
netdev_adjacent_add_links(struct net_device * dev)6425 static void netdev_adjacent_add_links(struct net_device *dev)
6426 {
6427 struct netdev_adjacent *iter;
6428
6429 struct net *net = dev_net(dev);
6430
6431 list_for_each_entry(iter, &dev->adj_list.upper, list) {
6432 if (!net_eq(net, dev_net(iter->dev)))
6433 continue;
6434 netdev_adjacent_sysfs_add(iter->dev, dev,
6435 &iter->dev->adj_list.lower);
6436 netdev_adjacent_sysfs_add(dev, iter->dev,
6437 &dev->adj_list.upper);
6438 }
6439
6440 list_for_each_entry(iter, &dev->adj_list.lower, list) {
6441 if (!net_eq(net, dev_net(iter->dev)))
6442 continue;
6443 netdev_adjacent_sysfs_add(iter->dev, dev,
6444 &iter->dev->adj_list.upper);
6445 netdev_adjacent_sysfs_add(dev, iter->dev,
6446 &dev->adj_list.lower);
6447 }
6448 }
6449
netdev_adjacent_del_links(struct net_device * dev)6450 static void netdev_adjacent_del_links(struct net_device *dev)
6451 {
6452 struct netdev_adjacent *iter;
6453
6454 struct net *net = dev_net(dev);
6455
6456 list_for_each_entry(iter, &dev->adj_list.upper, list) {
6457 if (!net_eq(net, dev_net(iter->dev)))
6458 continue;
6459 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6460 &iter->dev->adj_list.lower);
6461 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6462 &dev->adj_list.upper);
6463 }
6464
6465 list_for_each_entry(iter, &dev->adj_list.lower, list) {
6466 if (!net_eq(net, dev_net(iter->dev)))
6467 continue;
6468 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6469 &iter->dev->adj_list.upper);
6470 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6471 &dev->adj_list.lower);
6472 }
6473 }
6474
netdev_adjacent_rename_links(struct net_device * dev,char * oldname)6475 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
6476 {
6477 struct netdev_adjacent *iter;
6478
6479 struct net *net = dev_net(dev);
6480
6481 list_for_each_entry(iter, &dev->adj_list.upper, list) {
6482 if (!net_eq(net, dev_net(iter->dev)))
6483 continue;
6484 netdev_adjacent_sysfs_del(iter->dev, oldname,
6485 &iter->dev->adj_list.lower);
6486 netdev_adjacent_sysfs_add(iter->dev, dev,
6487 &iter->dev->adj_list.lower);
6488 }
6489
6490 list_for_each_entry(iter, &dev->adj_list.lower, list) {
6491 if (!net_eq(net, dev_net(iter->dev)))
6492 continue;
6493 netdev_adjacent_sysfs_del(iter->dev, oldname,
6494 &iter->dev->adj_list.upper);
6495 netdev_adjacent_sysfs_add(iter->dev, dev,
6496 &iter->dev->adj_list.upper);
6497 }
6498 }
6499
netdev_lower_dev_get_private(struct net_device * dev,struct net_device * lower_dev)6500 void *netdev_lower_dev_get_private(struct net_device *dev,
6501 struct net_device *lower_dev)
6502 {
6503 struct netdev_adjacent *lower;
6504
6505 if (!lower_dev)
6506 return NULL;
6507 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
6508 if (!lower)
6509 return NULL;
6510
6511 return lower->private;
6512 }
6513 EXPORT_SYMBOL(netdev_lower_dev_get_private);
6514
6515
dev_get_nest_level(struct net_device * dev)6516 int dev_get_nest_level(struct net_device *dev)
6517 {
6518 struct net_device *lower = NULL;
6519 struct list_head *iter;
6520 int max_nest = -1;
6521 int nest;
6522
6523 ASSERT_RTNL();
6524
6525 netdev_for_each_lower_dev(dev, lower, iter) {
6526 nest = dev_get_nest_level(lower);
6527 if (max_nest < nest)
6528 max_nest = nest;
6529 }
6530
6531 return max_nest + 1;
6532 }
6533 EXPORT_SYMBOL(dev_get_nest_level);
6534
6535 /**
6536 * netdev_lower_change - Dispatch event about lower device state change
6537 * @lower_dev: device
6538 * @lower_state_info: state to dispatch
6539 *
6540 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6541 * The caller must hold the RTNL lock.
6542 */
netdev_lower_state_changed(struct net_device * lower_dev,void * lower_state_info)6543 void netdev_lower_state_changed(struct net_device *lower_dev,
6544 void *lower_state_info)
6545 {
6546 struct netdev_notifier_changelowerstate_info changelowerstate_info;
6547
6548 ASSERT_RTNL();
6549 changelowerstate_info.lower_state_info = lower_state_info;
6550 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6551 &changelowerstate_info.info);
6552 }
6553 EXPORT_SYMBOL(netdev_lower_state_changed);
6554
dev_change_rx_flags(struct net_device * dev,int flags)6555 static void dev_change_rx_flags(struct net_device *dev, int flags)
6556 {
6557 const struct net_device_ops *ops = dev->netdev_ops;
6558
6559 if (ops->ndo_change_rx_flags)
6560 ops->ndo_change_rx_flags(dev, flags);
6561 }
6562
__dev_set_promiscuity(struct net_device * dev,int inc,bool notify)6563 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6564 {
6565 unsigned int old_flags = dev->flags;
6566 kuid_t uid;
6567 kgid_t gid;
6568
6569 ASSERT_RTNL();
6570
6571 dev->flags |= IFF_PROMISC;
6572 dev->promiscuity += inc;
6573 if (dev->promiscuity == 0) {
6574 /*
6575 * Avoid overflow.
6576 * If inc causes overflow, untouch promisc and return error.
6577 */
6578 if (inc < 0)
6579 dev->flags &= ~IFF_PROMISC;
6580 else {
6581 dev->promiscuity -= inc;
6582 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6583 dev->name);
6584 return -EOVERFLOW;
6585 }
6586 }
6587 if (dev->flags != old_flags) {
6588 pr_info("device %s %s promiscuous mode\n",
6589 dev->name,
6590 dev->flags & IFF_PROMISC ? "entered" : "left");
6591 if (audit_enabled) {
6592 current_uid_gid(&uid, &gid);
6593 audit_log(current->audit_context, GFP_ATOMIC,
6594 AUDIT_ANOM_PROMISCUOUS,
6595 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6596 dev->name, (dev->flags & IFF_PROMISC),
6597 (old_flags & IFF_PROMISC),
6598 from_kuid(&init_user_ns, audit_get_loginuid(current)),
6599 from_kuid(&init_user_ns, uid),
6600 from_kgid(&init_user_ns, gid),
6601 audit_get_sessionid(current));
6602 }
6603
6604 dev_change_rx_flags(dev, IFF_PROMISC);
6605 }
6606 if (notify)
6607 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
6608 return 0;
6609 }
6610
6611 /**
6612 * dev_set_promiscuity - update promiscuity count on a device
6613 * @dev: device
6614 * @inc: modifier
6615 *
6616 * Add or remove promiscuity from a device. While the count in the device
6617 * remains above zero the interface remains promiscuous. Once it hits zero
6618 * the device reverts back to normal filtering operation. A negative inc
6619 * value is used to drop promiscuity on the device.
6620 * Return 0 if successful or a negative errno code on error.
6621 */
dev_set_promiscuity(struct net_device * dev,int inc)6622 int dev_set_promiscuity(struct net_device *dev, int inc)
6623 {
6624 unsigned int old_flags = dev->flags;
6625 int err;
6626
6627 err = __dev_set_promiscuity(dev, inc, true);
6628 if (err < 0)
6629 return err;
6630 if (dev->flags != old_flags)
6631 dev_set_rx_mode(dev);
6632 return err;
6633 }
6634 EXPORT_SYMBOL(dev_set_promiscuity);
6635
__dev_set_allmulti(struct net_device * dev,int inc,bool notify)6636 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6637 {
6638 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6639
6640 ASSERT_RTNL();
6641
6642 dev->flags |= IFF_ALLMULTI;
6643 dev->allmulti += inc;
6644 if (dev->allmulti == 0) {
6645 /*
6646 * Avoid overflow.
6647 * If inc causes overflow, untouch allmulti and return error.
6648 */
6649 if (inc < 0)
6650 dev->flags &= ~IFF_ALLMULTI;
6651 else {
6652 dev->allmulti -= inc;
6653 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6654 dev->name);
6655 return -EOVERFLOW;
6656 }
6657 }
6658 if (dev->flags ^ old_flags) {
6659 dev_change_rx_flags(dev, IFF_ALLMULTI);
6660 dev_set_rx_mode(dev);
6661 if (notify)
6662 __dev_notify_flags(dev, old_flags,
6663 dev->gflags ^ old_gflags);
6664 }
6665 return 0;
6666 }
6667
6668 /**
6669 * dev_set_allmulti - update allmulti count on a device
6670 * @dev: device
6671 * @inc: modifier
6672 *
6673 * Add or remove reception of all multicast frames to a device. While the
6674 * count in the device remains above zero the interface remains listening
6675 * to all interfaces. Once it hits zero the device reverts back to normal
6676 * filtering operation. A negative @inc value is used to drop the counter
6677 * when releasing a resource needing all multicasts.
6678 * Return 0 if successful or a negative errno code on error.
6679 */
6680
dev_set_allmulti(struct net_device * dev,int inc)6681 int dev_set_allmulti(struct net_device *dev, int inc)
6682 {
6683 return __dev_set_allmulti(dev, inc, true);
6684 }
6685 EXPORT_SYMBOL(dev_set_allmulti);
6686
6687 /*
6688 * Upload unicast and multicast address lists to device and
6689 * configure RX filtering. When the device doesn't support unicast
6690 * filtering it is put in promiscuous mode while unicast addresses
6691 * are present.
6692 */
__dev_set_rx_mode(struct net_device * dev)6693 void __dev_set_rx_mode(struct net_device *dev)
6694 {
6695 const struct net_device_ops *ops = dev->netdev_ops;
6696
6697 /* dev_open will call this function so the list will stay sane. */
6698 if (!(dev->flags&IFF_UP))
6699 return;
6700
6701 if (!netif_device_present(dev))
6702 return;
6703
6704 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6705 /* Unicast addresses changes may only happen under the rtnl,
6706 * therefore calling __dev_set_promiscuity here is safe.
6707 */
6708 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6709 __dev_set_promiscuity(dev, 1, false);
6710 dev->uc_promisc = true;
6711 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6712 __dev_set_promiscuity(dev, -1, false);
6713 dev->uc_promisc = false;
6714 }
6715 }
6716
6717 if (ops->ndo_set_rx_mode)
6718 ops->ndo_set_rx_mode(dev);
6719 }
6720
dev_set_rx_mode(struct net_device * dev)6721 void dev_set_rx_mode(struct net_device *dev)
6722 {
6723 netif_addr_lock_bh(dev);
6724 __dev_set_rx_mode(dev);
6725 netif_addr_unlock_bh(dev);
6726 }
6727
6728 /**
6729 * dev_get_flags - get flags reported to userspace
6730 * @dev: device
6731 *
6732 * Get the combination of flag bits exported through APIs to userspace.
6733 */
dev_get_flags(const struct net_device * dev)6734 unsigned int dev_get_flags(const struct net_device *dev)
6735 {
6736 unsigned int flags;
6737
6738 flags = (dev->flags & ~(IFF_PROMISC |
6739 IFF_ALLMULTI |
6740 IFF_RUNNING |
6741 IFF_LOWER_UP |
6742 IFF_DORMANT)) |
6743 (dev->gflags & (IFF_PROMISC |
6744 IFF_ALLMULTI));
6745
6746 if (netif_running(dev)) {
6747 if (netif_oper_up(dev))
6748 flags |= IFF_RUNNING;
6749 if (netif_carrier_ok(dev))
6750 flags |= IFF_LOWER_UP;
6751 if (netif_dormant(dev))
6752 flags |= IFF_DORMANT;
6753 }
6754
6755 return flags;
6756 }
6757 EXPORT_SYMBOL(dev_get_flags);
6758
__dev_change_flags(struct net_device * dev,unsigned int flags)6759 int __dev_change_flags(struct net_device *dev, unsigned int flags)
6760 {
6761 unsigned int old_flags = dev->flags;
6762 int ret;
6763
6764 ASSERT_RTNL();
6765
6766 /*
6767 * Set the flags on our device.
6768 */
6769
6770 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6771 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6772 IFF_AUTOMEDIA)) |
6773 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6774 IFF_ALLMULTI));
6775
6776 /*
6777 * Load in the correct multicast list now the flags have changed.
6778 */
6779
6780 if ((old_flags ^ flags) & IFF_MULTICAST)
6781 dev_change_rx_flags(dev, IFF_MULTICAST);
6782
6783 dev_set_rx_mode(dev);
6784
6785 /*
6786 * Have we downed the interface. We handle IFF_UP ourselves
6787 * according to user attempts to set it, rather than blindly
6788 * setting it.
6789 */
6790
6791 ret = 0;
6792 if ((old_flags ^ flags) & IFF_UP) {
6793 if (old_flags & IFF_UP)
6794 __dev_close(dev);
6795 else
6796 ret = __dev_open(dev);
6797 }
6798
6799 if ((flags ^ dev->gflags) & IFF_PROMISC) {
6800 int inc = (flags & IFF_PROMISC) ? 1 : -1;
6801 unsigned int old_flags = dev->flags;
6802
6803 dev->gflags ^= IFF_PROMISC;
6804
6805 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6806 if (dev->flags != old_flags)
6807 dev_set_rx_mode(dev);
6808 }
6809
6810 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6811 * is important. Some (broken) drivers set IFF_PROMISC, when
6812 * IFF_ALLMULTI is requested not asking us and not reporting.
6813 */
6814 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6815 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6816
6817 dev->gflags ^= IFF_ALLMULTI;
6818 __dev_set_allmulti(dev, inc, false);
6819 }
6820
6821 return ret;
6822 }
6823
__dev_notify_flags(struct net_device * dev,unsigned int old_flags,unsigned int gchanges)6824 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6825 unsigned int gchanges)
6826 {
6827 unsigned int changes = dev->flags ^ old_flags;
6828
6829 if (gchanges)
6830 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6831
6832 if (changes & IFF_UP) {
6833 if (dev->flags & IFF_UP)
6834 call_netdevice_notifiers(NETDEV_UP, dev);
6835 else
6836 call_netdevice_notifiers(NETDEV_DOWN, dev);
6837 }
6838
6839 if (dev->flags & IFF_UP &&
6840 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6841 struct netdev_notifier_change_info change_info;
6842
6843 change_info.flags_changed = changes;
6844 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6845 &change_info.info);
6846 }
6847 }
6848
6849 /**
6850 * dev_change_flags - change device settings
6851 * @dev: device
6852 * @flags: device state flags
6853 *
6854 * Change settings on device based state flags. The flags are
6855 * in the userspace exported format.
6856 */
dev_change_flags(struct net_device * dev,unsigned int flags)6857 int dev_change_flags(struct net_device *dev, unsigned int flags)
6858 {
6859 int ret;
6860 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6861
6862 ret = __dev_change_flags(dev, flags);
6863 if (ret < 0)
6864 return ret;
6865
6866 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6867 __dev_notify_flags(dev, old_flags, changes);
6868 return ret;
6869 }
6870 EXPORT_SYMBOL(dev_change_flags);
6871
__dev_set_mtu(struct net_device * dev,int new_mtu)6872 int __dev_set_mtu(struct net_device *dev, int new_mtu)
6873 {
6874 const struct net_device_ops *ops = dev->netdev_ops;
6875
6876 if (ops->ndo_change_mtu)
6877 return ops->ndo_change_mtu(dev, new_mtu);
6878
6879 /* Pairs with all the lockless reads of dev->mtu in the stack */
6880 WRITE_ONCE(dev->mtu, new_mtu);
6881 return 0;
6882 }
6883 EXPORT_SYMBOL(__dev_set_mtu);
6884
6885 /**
6886 * dev_set_mtu - Change maximum transfer unit
6887 * @dev: device
6888 * @new_mtu: new transfer unit
6889 *
6890 * Change the maximum transfer size of the network device.
6891 */
dev_set_mtu(struct net_device * dev,int new_mtu)6892 int dev_set_mtu(struct net_device *dev, int new_mtu)
6893 {
6894 int err, orig_mtu;
6895
6896 if (new_mtu == dev->mtu)
6897 return 0;
6898
6899 err = dev_validate_mtu(dev, new_mtu);
6900 if (err)
6901 return err;
6902
6903 if (!netif_device_present(dev))
6904 return -ENODEV;
6905
6906 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6907 err = notifier_to_errno(err);
6908 if (err)
6909 return err;
6910
6911 orig_mtu = dev->mtu;
6912 err = __dev_set_mtu(dev, new_mtu);
6913
6914 if (!err) {
6915 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
6916 orig_mtu);
6917 err = notifier_to_errno(err);
6918 if (err) {
6919 /* setting mtu back and notifying everyone again,
6920 * so that they have a chance to revert changes.
6921 */
6922 __dev_set_mtu(dev, orig_mtu);
6923 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
6924 new_mtu);
6925 }
6926 }
6927 return err;
6928 }
6929 EXPORT_SYMBOL(dev_set_mtu);
6930
6931 /**
6932 * dev_set_group - Change group this device belongs to
6933 * @dev: device
6934 * @new_group: group this device should belong to
6935 */
dev_set_group(struct net_device * dev,int new_group)6936 void dev_set_group(struct net_device *dev, int new_group)
6937 {
6938 dev->group = new_group;
6939 }
6940 EXPORT_SYMBOL(dev_set_group);
6941
6942 /**
6943 * dev_set_mac_address - Change Media Access Control Address
6944 * @dev: device
6945 * @sa: new address
6946 *
6947 * Change the hardware (MAC) address of the device
6948 */
dev_set_mac_address(struct net_device * dev,struct sockaddr * sa)6949 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6950 {
6951 const struct net_device_ops *ops = dev->netdev_ops;
6952 int err;
6953
6954 if (!ops->ndo_set_mac_address)
6955 return -EOPNOTSUPP;
6956 if (sa->sa_family != dev->type)
6957 return -EINVAL;
6958 if (!netif_device_present(dev))
6959 return -ENODEV;
6960 err = ops->ndo_set_mac_address(dev, sa);
6961 if (err)
6962 return err;
6963 dev->addr_assign_type = NET_ADDR_SET;
6964 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6965 add_device_randomness(dev->dev_addr, dev->addr_len);
6966 return 0;
6967 }
6968 EXPORT_SYMBOL(dev_set_mac_address);
6969
6970 /**
6971 * dev_change_carrier - Change device carrier
6972 * @dev: device
6973 * @new_carrier: new value
6974 *
6975 * Change device carrier
6976 */
dev_change_carrier(struct net_device * dev,bool new_carrier)6977 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6978 {
6979 const struct net_device_ops *ops = dev->netdev_ops;
6980
6981 if (!ops->ndo_change_carrier)
6982 return -EOPNOTSUPP;
6983 if (!netif_device_present(dev))
6984 return -ENODEV;
6985 return ops->ndo_change_carrier(dev, new_carrier);
6986 }
6987 EXPORT_SYMBOL(dev_change_carrier);
6988
6989 /**
6990 * dev_get_phys_port_id - Get device physical port ID
6991 * @dev: device
6992 * @ppid: port ID
6993 *
6994 * Get device physical port ID
6995 */
dev_get_phys_port_id(struct net_device * dev,struct netdev_phys_item_id * ppid)6996 int dev_get_phys_port_id(struct net_device *dev,
6997 struct netdev_phys_item_id *ppid)
6998 {
6999 const struct net_device_ops *ops = dev->netdev_ops;
7000
7001 if (!ops->ndo_get_phys_port_id)
7002 return -EOPNOTSUPP;
7003 return ops->ndo_get_phys_port_id(dev, ppid);
7004 }
7005 EXPORT_SYMBOL(dev_get_phys_port_id);
7006
7007 /**
7008 * dev_get_phys_port_name - Get device physical port name
7009 * @dev: device
7010 * @name: port name
7011 * @len: limit of bytes to copy to name
7012 *
7013 * Get device physical port name
7014 */
dev_get_phys_port_name(struct net_device * dev,char * name,size_t len)7015 int dev_get_phys_port_name(struct net_device *dev,
7016 char *name, size_t len)
7017 {
7018 const struct net_device_ops *ops = dev->netdev_ops;
7019
7020 if (!ops->ndo_get_phys_port_name)
7021 return -EOPNOTSUPP;
7022 return ops->ndo_get_phys_port_name(dev, name, len);
7023 }
7024 EXPORT_SYMBOL(dev_get_phys_port_name);
7025
7026 /**
7027 * dev_change_proto_down - update protocol port state information
7028 * @dev: device
7029 * @proto_down: new value
7030 *
7031 * This info can be used by switch drivers to set the phys state of the
7032 * port.
7033 */
dev_change_proto_down(struct net_device * dev,bool proto_down)7034 int dev_change_proto_down(struct net_device *dev, bool proto_down)
7035 {
7036 const struct net_device_ops *ops = dev->netdev_ops;
7037
7038 if (!ops->ndo_change_proto_down)
7039 return -EOPNOTSUPP;
7040 if (!netif_device_present(dev))
7041 return -ENODEV;
7042 return ops->ndo_change_proto_down(dev, proto_down);
7043 }
7044 EXPORT_SYMBOL(dev_change_proto_down);
7045
__dev_xdp_attached(struct net_device * dev,xdp_op_t xdp_op,u32 * prog_id)7046 u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id)
7047 {
7048 struct netdev_xdp xdp;
7049
7050 memset(&xdp, 0, sizeof(xdp));
7051 xdp.command = XDP_QUERY_PROG;
7052
7053 /* Query must always succeed. */
7054 WARN_ON(xdp_op(dev, &xdp) < 0);
7055 if (prog_id)
7056 *prog_id = xdp.prog_id;
7057
7058 return xdp.prog_attached;
7059 }
7060
dev_xdp_install(struct net_device * dev,xdp_op_t xdp_op,struct netlink_ext_ack * extack,u32 flags,struct bpf_prog * prog)7061 static int dev_xdp_install(struct net_device *dev, xdp_op_t xdp_op,
7062 struct netlink_ext_ack *extack, u32 flags,
7063 struct bpf_prog *prog)
7064 {
7065 struct netdev_xdp xdp;
7066
7067 memset(&xdp, 0, sizeof(xdp));
7068 if (flags & XDP_FLAGS_HW_MODE)
7069 xdp.command = XDP_SETUP_PROG_HW;
7070 else
7071 xdp.command = XDP_SETUP_PROG;
7072 xdp.extack = extack;
7073 xdp.flags = flags;
7074 xdp.prog = prog;
7075
7076 return xdp_op(dev, &xdp);
7077 }
7078
7079 /**
7080 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
7081 * @dev: device
7082 * @extack: netlink extended ack
7083 * @fd: new program fd or negative value to clear
7084 * @flags: xdp-related flags
7085 *
7086 * Set or clear a bpf program for a device
7087 */
dev_change_xdp_fd(struct net_device * dev,struct netlink_ext_ack * extack,int fd,u32 flags)7088 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
7089 int fd, u32 flags)
7090 {
7091 const struct net_device_ops *ops = dev->netdev_ops;
7092 struct bpf_prog *prog = NULL;
7093 xdp_op_t xdp_op, xdp_chk;
7094 int err;
7095
7096 ASSERT_RTNL();
7097
7098 xdp_op = xdp_chk = ops->ndo_xdp;
7099 if (!xdp_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
7100 return -EOPNOTSUPP;
7101 if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
7102 xdp_op = generic_xdp_install;
7103 if (xdp_op == xdp_chk)
7104 xdp_chk = generic_xdp_install;
7105
7106 if (fd >= 0) {
7107 if (xdp_chk && __dev_xdp_attached(dev, xdp_chk, NULL))
7108 return -EEXIST;
7109 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
7110 __dev_xdp_attached(dev, xdp_op, NULL))
7111 return -EBUSY;
7112
7113 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
7114 if (IS_ERR(prog))
7115 return PTR_ERR(prog);
7116 }
7117
7118 err = dev_xdp_install(dev, xdp_op, extack, flags, prog);
7119 if (err < 0 && prog)
7120 bpf_prog_put(prog);
7121
7122 return err;
7123 }
7124
7125 /**
7126 * dev_new_index - allocate an ifindex
7127 * @net: the applicable net namespace
7128 *
7129 * Returns a suitable unique value for a new device interface
7130 * number. The caller must hold the rtnl semaphore or the
7131 * dev_base_lock to be sure it remains unique.
7132 */
dev_new_index(struct net * net)7133 static int dev_new_index(struct net *net)
7134 {
7135 int ifindex = net->ifindex;
7136
7137 for (;;) {
7138 if (++ifindex <= 0)
7139 ifindex = 1;
7140 if (!__dev_get_by_index(net, ifindex))
7141 return net->ifindex = ifindex;
7142 }
7143 }
7144
7145 /* Delayed registration/unregisteration */
7146 static LIST_HEAD(net_todo_list);
7147 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
7148
net_set_todo(struct net_device * dev)7149 static void net_set_todo(struct net_device *dev)
7150 {
7151 list_add_tail(&dev->todo_list, &net_todo_list);
7152 dev_net(dev)->dev_unreg_count++;
7153 }
7154
rollback_registered_many(struct list_head * head)7155 static void rollback_registered_many(struct list_head *head)
7156 {
7157 struct net_device *dev, *tmp;
7158 LIST_HEAD(close_head);
7159
7160 BUG_ON(dev_boot_phase);
7161 ASSERT_RTNL();
7162
7163 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
7164 /* Some devices call without registering
7165 * for initialization unwind. Remove those
7166 * devices and proceed with the remaining.
7167 */
7168 if (dev->reg_state == NETREG_UNINITIALIZED) {
7169 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
7170 dev->name, dev);
7171
7172 WARN_ON(1);
7173 list_del(&dev->unreg_list);
7174 continue;
7175 }
7176 dev->dismantle = true;
7177 BUG_ON(dev->reg_state != NETREG_REGISTERED);
7178 }
7179
7180 /* If device is running, close it first. */
7181 list_for_each_entry(dev, head, unreg_list)
7182 list_add_tail(&dev->close_list, &close_head);
7183 dev_close_many(&close_head, true);
7184
7185 list_for_each_entry(dev, head, unreg_list) {
7186 /* And unlink it from device chain. */
7187 unlist_netdevice(dev);
7188
7189 dev->reg_state = NETREG_UNREGISTERING;
7190 }
7191 flush_all_backlogs();
7192
7193 synchronize_net();
7194
7195 list_for_each_entry(dev, head, unreg_list) {
7196 struct sk_buff *skb = NULL;
7197
7198 /* Shutdown queueing discipline. */
7199 dev_shutdown(dev);
7200
7201
7202 /* Notify protocols, that we are about to destroy
7203 * this device. They should clean all the things.
7204 */
7205 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7206
7207 if (!dev->rtnl_link_ops ||
7208 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7209 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
7210 GFP_KERNEL);
7211
7212 /*
7213 * Flush the unicast and multicast chains
7214 */
7215 dev_uc_flush(dev);
7216 dev_mc_flush(dev);
7217
7218 if (dev->netdev_ops->ndo_uninit)
7219 dev->netdev_ops->ndo_uninit(dev);
7220
7221 if (skb)
7222 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
7223
7224 /* Notifier chain MUST detach us all upper devices. */
7225 WARN_ON(netdev_has_any_upper_dev(dev));
7226 WARN_ON(netdev_has_any_lower_dev(dev));
7227
7228 /* Remove entries from kobject tree */
7229 netdev_unregister_kobject(dev);
7230 #ifdef CONFIG_XPS
7231 /* Remove XPS queueing entries */
7232 netif_reset_xps_queues_gt(dev, 0);
7233 #endif
7234 }
7235
7236 synchronize_net();
7237
7238 list_for_each_entry(dev, head, unreg_list)
7239 dev_put(dev);
7240 }
7241
rollback_registered(struct net_device * dev)7242 static void rollback_registered(struct net_device *dev)
7243 {
7244 LIST_HEAD(single);
7245
7246 list_add(&dev->unreg_list, &single);
7247 rollback_registered_many(&single);
7248 list_del(&single);
7249 }
7250
netdev_sync_upper_features(struct net_device * lower,struct net_device * upper,netdev_features_t features)7251 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
7252 struct net_device *upper, netdev_features_t features)
7253 {
7254 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7255 netdev_features_t feature;
7256 int feature_bit;
7257
7258 for_each_netdev_feature(upper_disables, feature_bit) {
7259 feature = __NETIF_F_BIT(feature_bit);
7260 if (!(upper->wanted_features & feature)
7261 && (features & feature)) {
7262 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
7263 &feature, upper->name);
7264 features &= ~feature;
7265 }
7266 }
7267
7268 return features;
7269 }
7270
netdev_sync_lower_features(struct net_device * upper,struct net_device * lower,netdev_features_t features)7271 static void netdev_sync_lower_features(struct net_device *upper,
7272 struct net_device *lower, netdev_features_t features)
7273 {
7274 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7275 netdev_features_t feature;
7276 int feature_bit;
7277
7278 for_each_netdev_feature(upper_disables, feature_bit) {
7279 feature = __NETIF_F_BIT(feature_bit);
7280 if (!(features & feature) && (lower->features & feature)) {
7281 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
7282 &feature, lower->name);
7283 lower->wanted_features &= ~feature;
7284 netdev_update_features(lower);
7285
7286 if (unlikely(lower->features & feature))
7287 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
7288 &feature, lower->name);
7289 }
7290 }
7291 }
7292
netdev_fix_features(struct net_device * dev,netdev_features_t features)7293 static netdev_features_t netdev_fix_features(struct net_device *dev,
7294 netdev_features_t features)
7295 {
7296 /* Fix illegal checksum combinations */
7297 if ((features & NETIF_F_HW_CSUM) &&
7298 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
7299 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
7300 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
7301 }
7302
7303 /* TSO requires that SG is present as well. */
7304 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
7305 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
7306 features &= ~NETIF_F_ALL_TSO;
7307 }
7308
7309 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
7310 !(features & NETIF_F_IP_CSUM)) {
7311 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
7312 features &= ~NETIF_F_TSO;
7313 features &= ~NETIF_F_TSO_ECN;
7314 }
7315
7316 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
7317 !(features & NETIF_F_IPV6_CSUM)) {
7318 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
7319 features &= ~NETIF_F_TSO6;
7320 }
7321
7322 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
7323 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
7324 features &= ~NETIF_F_TSO_MANGLEID;
7325
7326 /* TSO ECN requires that TSO is present as well. */
7327 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
7328 features &= ~NETIF_F_TSO_ECN;
7329
7330 /* Software GSO depends on SG. */
7331 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
7332 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
7333 features &= ~NETIF_F_GSO;
7334 }
7335
7336 /* GSO partial features require GSO partial be set */
7337 if ((features & dev->gso_partial_features) &&
7338 !(features & NETIF_F_GSO_PARTIAL)) {
7339 netdev_dbg(dev,
7340 "Dropping partially supported GSO features since no GSO partial.\n");
7341 features &= ~dev->gso_partial_features;
7342 }
7343
7344 return features;
7345 }
7346
__netdev_update_features(struct net_device * dev)7347 int __netdev_update_features(struct net_device *dev)
7348 {
7349 struct net_device *upper, *lower;
7350 netdev_features_t features;
7351 struct list_head *iter;
7352 int err = -1;
7353
7354 ASSERT_RTNL();
7355
7356 features = netdev_get_wanted_features(dev);
7357
7358 if (dev->netdev_ops->ndo_fix_features)
7359 features = dev->netdev_ops->ndo_fix_features(dev, features);
7360
7361 /* driver might be less strict about feature dependencies */
7362 features = netdev_fix_features(dev, features);
7363
7364 /* some features can't be enabled if they're off an an upper device */
7365 netdev_for_each_upper_dev_rcu(dev, upper, iter)
7366 features = netdev_sync_upper_features(dev, upper, features);
7367
7368 if (dev->features == features)
7369 goto sync_lower;
7370
7371 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7372 &dev->features, &features);
7373
7374 if (dev->netdev_ops->ndo_set_features)
7375 err = dev->netdev_ops->ndo_set_features(dev, features);
7376 else
7377 err = 0;
7378
7379 if (unlikely(err < 0)) {
7380 netdev_err(dev,
7381 "set_features() failed (%d); wanted %pNF, left %pNF\n",
7382 err, &features, &dev->features);
7383 /* return non-0 since some features might have changed and
7384 * it's better to fire a spurious notification than miss it
7385 */
7386 return -1;
7387 }
7388
7389 sync_lower:
7390 /* some features must be disabled on lower devices when disabled
7391 * on an upper device (think: bonding master or bridge)
7392 */
7393 netdev_for_each_lower_dev(dev, lower, iter)
7394 netdev_sync_lower_features(dev, lower, features);
7395
7396 if (!err) {
7397 netdev_features_t diff = features ^ dev->features;
7398
7399 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
7400 /* udp_tunnel_{get,drop}_rx_info both need
7401 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
7402 * device, or they won't do anything.
7403 * Thus we need to update dev->features
7404 * *before* calling udp_tunnel_get_rx_info,
7405 * but *after* calling udp_tunnel_drop_rx_info.
7406 */
7407 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
7408 dev->features = features;
7409 udp_tunnel_get_rx_info(dev);
7410 } else {
7411 udp_tunnel_drop_rx_info(dev);
7412 }
7413 }
7414
7415 dev->features = features;
7416 }
7417
7418 return err < 0 ? 0 : 1;
7419 }
7420
7421 /**
7422 * netdev_update_features - recalculate device features
7423 * @dev: the device to check
7424 *
7425 * Recalculate dev->features set and send notifications if it
7426 * has changed. Should be called after driver or hardware dependent
7427 * conditions might have changed that influence the features.
7428 */
netdev_update_features(struct net_device * dev)7429 void netdev_update_features(struct net_device *dev)
7430 {
7431 if (__netdev_update_features(dev))
7432 netdev_features_change(dev);
7433 }
7434 EXPORT_SYMBOL(netdev_update_features);
7435
7436 /**
7437 * netdev_change_features - recalculate device features
7438 * @dev: the device to check
7439 *
7440 * Recalculate dev->features set and send notifications even
7441 * if they have not changed. Should be called instead of
7442 * netdev_update_features() if also dev->vlan_features might
7443 * have changed to allow the changes to be propagated to stacked
7444 * VLAN devices.
7445 */
netdev_change_features(struct net_device * dev)7446 void netdev_change_features(struct net_device *dev)
7447 {
7448 __netdev_update_features(dev);
7449 netdev_features_change(dev);
7450 }
7451 EXPORT_SYMBOL(netdev_change_features);
7452
7453 /**
7454 * netif_stacked_transfer_operstate - transfer operstate
7455 * @rootdev: the root or lower level device to transfer state from
7456 * @dev: the device to transfer operstate to
7457 *
7458 * Transfer operational state from root to device. This is normally
7459 * called when a stacking relationship exists between the root
7460 * device and the device(a leaf device).
7461 */
netif_stacked_transfer_operstate(const struct net_device * rootdev,struct net_device * dev)7462 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7463 struct net_device *dev)
7464 {
7465 if (rootdev->operstate == IF_OPER_DORMANT)
7466 netif_dormant_on(dev);
7467 else
7468 netif_dormant_off(dev);
7469
7470 if (netif_carrier_ok(rootdev))
7471 netif_carrier_on(dev);
7472 else
7473 netif_carrier_off(dev);
7474 }
7475 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7476
7477 #ifdef CONFIG_SYSFS
netif_alloc_rx_queues(struct net_device * dev)7478 static int netif_alloc_rx_queues(struct net_device *dev)
7479 {
7480 unsigned int i, count = dev->num_rx_queues;
7481 struct netdev_rx_queue *rx;
7482 size_t sz = count * sizeof(*rx);
7483
7484 BUG_ON(count < 1);
7485
7486 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
7487 if (!rx)
7488 return -ENOMEM;
7489
7490 dev->_rx = rx;
7491
7492 for (i = 0; i < count; i++)
7493 rx[i].dev = dev;
7494 return 0;
7495 }
7496 #endif
7497
netdev_init_one_queue(struct net_device * dev,struct netdev_queue * queue,void * _unused)7498 static void netdev_init_one_queue(struct net_device *dev,
7499 struct netdev_queue *queue, void *_unused)
7500 {
7501 /* Initialize queue lock */
7502 spin_lock_init(&queue->_xmit_lock);
7503 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7504 queue->xmit_lock_owner = -1;
7505 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
7506 queue->dev = dev;
7507 #ifdef CONFIG_BQL
7508 dql_init(&queue->dql, HZ);
7509 #endif
7510 }
7511
netif_free_tx_queues(struct net_device * dev)7512 static void netif_free_tx_queues(struct net_device *dev)
7513 {
7514 kvfree(dev->_tx);
7515 }
7516
netif_alloc_netdev_queues(struct net_device * dev)7517 static int netif_alloc_netdev_queues(struct net_device *dev)
7518 {
7519 unsigned int count = dev->num_tx_queues;
7520 struct netdev_queue *tx;
7521 size_t sz = count * sizeof(*tx);
7522
7523 if (count < 1 || count > 0xffff)
7524 return -EINVAL;
7525
7526 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
7527 if (!tx)
7528 return -ENOMEM;
7529
7530 dev->_tx = tx;
7531
7532 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7533 spin_lock_init(&dev->tx_global_lock);
7534
7535 return 0;
7536 }
7537
netif_tx_stop_all_queues(struct net_device * dev)7538 void netif_tx_stop_all_queues(struct net_device *dev)
7539 {
7540 unsigned int i;
7541
7542 for (i = 0; i < dev->num_tx_queues; i++) {
7543 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
7544
7545 netif_tx_stop_queue(txq);
7546 }
7547 }
7548 EXPORT_SYMBOL(netif_tx_stop_all_queues);
7549
7550 /**
7551 * register_netdevice - register a network device
7552 * @dev: device to register
7553 *
7554 * Take a completed network device structure and add it to the kernel
7555 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7556 * chain. 0 is returned on success. A negative errno code is returned
7557 * on a failure to set up the device, or if the name is a duplicate.
7558 *
7559 * Callers must hold the rtnl semaphore. You may want
7560 * register_netdev() instead of this.
7561 *
7562 * BUGS:
7563 * The locking appears insufficient to guarantee two parallel registers
7564 * will not get the same name.
7565 */
7566
register_netdevice(struct net_device * dev)7567 int register_netdevice(struct net_device *dev)
7568 {
7569 int ret;
7570 struct net *net = dev_net(dev);
7571
7572 BUG_ON(dev_boot_phase);
7573 ASSERT_RTNL();
7574
7575 might_sleep();
7576
7577 /* When net_device's are persistent, this will be fatal. */
7578 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
7579 BUG_ON(!net);
7580
7581 spin_lock_init(&dev->addr_list_lock);
7582 netdev_set_addr_lockdep_class(dev);
7583
7584 ret = dev_get_valid_name(net, dev, dev->name);
7585 if (ret < 0)
7586 goto out;
7587
7588 /* Init, if this function is available */
7589 if (dev->netdev_ops->ndo_init) {
7590 ret = dev->netdev_ops->ndo_init(dev);
7591 if (ret) {
7592 if (ret > 0)
7593 ret = -EIO;
7594 goto out;
7595 }
7596 }
7597
7598 if (((dev->hw_features | dev->features) &
7599 NETIF_F_HW_VLAN_CTAG_FILTER) &&
7600 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7601 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7602 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7603 ret = -EINVAL;
7604 goto err_uninit;
7605 }
7606
7607 ret = -EBUSY;
7608 if (!dev->ifindex)
7609 dev->ifindex = dev_new_index(net);
7610 else if (__dev_get_by_index(net, dev->ifindex))
7611 goto err_uninit;
7612
7613 /* Transfer changeable features to wanted_features and enable
7614 * software offloads (GSO and GRO).
7615 */
7616 dev->hw_features |= NETIF_F_SOFT_FEATURES;
7617 dev->features |= NETIF_F_SOFT_FEATURES;
7618
7619 if (dev->netdev_ops->ndo_udp_tunnel_add) {
7620 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7621 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7622 }
7623
7624 dev->wanted_features = dev->features & dev->hw_features;
7625
7626 if (!(dev->flags & IFF_LOOPBACK))
7627 dev->hw_features |= NETIF_F_NOCACHE_COPY;
7628
7629 /* If IPv4 TCP segmentation offload is supported we should also
7630 * allow the device to enable segmenting the frame with the option
7631 * of ignoring a static IP ID value. This doesn't enable the
7632 * feature itself but allows the user to enable it later.
7633 */
7634 if (dev->hw_features & NETIF_F_TSO)
7635 dev->hw_features |= NETIF_F_TSO_MANGLEID;
7636 if (dev->vlan_features & NETIF_F_TSO)
7637 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7638 if (dev->mpls_features & NETIF_F_TSO)
7639 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7640 if (dev->hw_enc_features & NETIF_F_TSO)
7641 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
7642
7643 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
7644 */
7645 dev->vlan_features |= NETIF_F_HIGHDMA;
7646
7647 /* Make NETIF_F_SG inheritable to tunnel devices.
7648 */
7649 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
7650
7651 /* Make NETIF_F_SG inheritable to MPLS.
7652 */
7653 dev->mpls_features |= NETIF_F_SG;
7654
7655 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7656 ret = notifier_to_errno(ret);
7657 if (ret)
7658 goto err_uninit;
7659
7660 ret = netdev_register_kobject(dev);
7661 if (ret) {
7662 dev->reg_state = NETREG_UNREGISTERED;
7663 goto err_uninit;
7664 }
7665 dev->reg_state = NETREG_REGISTERED;
7666
7667 __netdev_update_features(dev);
7668
7669 /*
7670 * Default initial state at registry is that the
7671 * device is present.
7672 */
7673
7674 set_bit(__LINK_STATE_PRESENT, &dev->state);
7675
7676 linkwatch_init_dev(dev);
7677
7678 dev_init_scheduler(dev);
7679 dev_hold(dev);
7680 list_netdevice(dev);
7681 add_device_randomness(dev->dev_addr, dev->addr_len);
7682
7683 /* If the device has permanent device address, driver should
7684 * set dev_addr and also addr_assign_type should be set to
7685 * NET_ADDR_PERM (default value).
7686 */
7687 if (dev->addr_assign_type == NET_ADDR_PERM)
7688 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7689
7690 /* Notify protocols, that a new device appeared. */
7691 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7692 ret = notifier_to_errno(ret);
7693 if (ret) {
7694 rollback_registered(dev);
7695 rcu_barrier();
7696
7697 dev->reg_state = NETREG_UNREGISTERED;
7698 }
7699 /*
7700 * Prevent userspace races by waiting until the network
7701 * device is fully setup before sending notifications.
7702 */
7703 if (!dev->rtnl_link_ops ||
7704 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7705 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7706
7707 out:
7708 return ret;
7709
7710 err_uninit:
7711 if (dev->netdev_ops->ndo_uninit)
7712 dev->netdev_ops->ndo_uninit(dev);
7713 if (dev->priv_destructor)
7714 dev->priv_destructor(dev);
7715 goto out;
7716 }
7717 EXPORT_SYMBOL(register_netdevice);
7718
7719 /**
7720 * init_dummy_netdev - init a dummy network device for NAPI
7721 * @dev: device to init
7722 *
7723 * This takes a network device structure and initialize the minimum
7724 * amount of fields so it can be used to schedule NAPI polls without
7725 * registering a full blown interface. This is to be used by drivers
7726 * that need to tie several hardware interfaces to a single NAPI
7727 * poll scheduler due to HW limitations.
7728 */
init_dummy_netdev(struct net_device * dev)7729 int init_dummy_netdev(struct net_device *dev)
7730 {
7731 /* Clear everything. Note we don't initialize spinlocks
7732 * are they aren't supposed to be taken by any of the
7733 * NAPI code and this dummy netdev is supposed to be
7734 * only ever used for NAPI polls
7735 */
7736 memset(dev, 0, sizeof(struct net_device));
7737
7738 /* make sure we BUG if trying to hit standard
7739 * register/unregister code path
7740 */
7741 dev->reg_state = NETREG_DUMMY;
7742
7743 /* NAPI wants this */
7744 INIT_LIST_HEAD(&dev->napi_list);
7745
7746 /* a dummy interface is started by default */
7747 set_bit(__LINK_STATE_PRESENT, &dev->state);
7748 set_bit(__LINK_STATE_START, &dev->state);
7749
7750 /* napi_busy_loop stats accounting wants this */
7751 dev_net_set(dev, &init_net);
7752
7753 /* Note : We dont allocate pcpu_refcnt for dummy devices,
7754 * because users of this 'device' dont need to change
7755 * its refcount.
7756 */
7757
7758 return 0;
7759 }
7760 EXPORT_SYMBOL_GPL(init_dummy_netdev);
7761
7762
dev_validate_mtu(struct net_device * dev,int new_mtu)7763 int dev_validate_mtu(struct net_device *dev, int new_mtu)
7764 {
7765 /* MTU must be positive, and in range */
7766 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
7767 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
7768 dev->name, new_mtu, dev->min_mtu);
7769 return -EINVAL;
7770 }
7771
7772 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
7773 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
7774 dev->name, new_mtu, dev->max_mtu);
7775 return -EINVAL;
7776 }
7777 return 0;
7778 }
7779
7780 /**
7781 * register_netdev - register a network device
7782 * @dev: device to register
7783 *
7784 * Take a completed network device structure and add it to the kernel
7785 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7786 * chain. 0 is returned on success. A negative errno code is returned
7787 * on a failure to set up the device, or if the name is a duplicate.
7788 *
7789 * This is a wrapper around register_netdevice that takes the rtnl semaphore
7790 * and expands the device name if you passed a format string to
7791 * alloc_netdev.
7792 */
register_netdev(struct net_device * dev)7793 int register_netdev(struct net_device *dev)
7794 {
7795 int err;
7796
7797 rtnl_lock();
7798 err = register_netdevice(dev);
7799 rtnl_unlock();
7800 return err;
7801 }
7802 EXPORT_SYMBOL(register_netdev);
7803
netdev_refcnt_read(const struct net_device * dev)7804 int netdev_refcnt_read(const struct net_device *dev)
7805 {
7806 int i, refcnt = 0;
7807
7808 for_each_possible_cpu(i)
7809 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7810 return refcnt;
7811 }
7812 EXPORT_SYMBOL(netdev_refcnt_read);
7813
7814 /**
7815 * netdev_wait_allrefs - wait until all references are gone.
7816 * @dev: target net_device
7817 *
7818 * This is called when unregistering network devices.
7819 *
7820 * Any protocol or device that holds a reference should register
7821 * for netdevice notification, and cleanup and put back the
7822 * reference if they receive an UNREGISTER event.
7823 * We can get stuck here if buggy protocols don't correctly
7824 * call dev_put.
7825 */
netdev_wait_allrefs(struct net_device * dev)7826 static void netdev_wait_allrefs(struct net_device *dev)
7827 {
7828 unsigned long rebroadcast_time, warning_time;
7829 int refcnt;
7830
7831 linkwatch_forget_dev(dev);
7832
7833 rebroadcast_time = warning_time = jiffies;
7834 refcnt = netdev_refcnt_read(dev);
7835
7836 while (refcnt != 0) {
7837 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7838 rtnl_lock();
7839
7840 /* Rebroadcast unregister notification */
7841 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7842
7843 __rtnl_unlock();
7844 rcu_barrier();
7845 rtnl_lock();
7846
7847 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7848 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7849 &dev->state)) {
7850 /* We must not have linkwatch events
7851 * pending on unregister. If this
7852 * happens, we simply run the queue
7853 * unscheduled, resulting in a noop
7854 * for this device.
7855 */
7856 linkwatch_run_queue();
7857 }
7858
7859 __rtnl_unlock();
7860
7861 rebroadcast_time = jiffies;
7862 }
7863
7864 msleep(250);
7865
7866 refcnt = netdev_refcnt_read(dev);
7867
7868 if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) {
7869 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7870 dev->name, refcnt);
7871 warning_time = jiffies;
7872 }
7873 }
7874 }
7875
7876 /* The sequence is:
7877 *
7878 * rtnl_lock();
7879 * ...
7880 * register_netdevice(x1);
7881 * register_netdevice(x2);
7882 * ...
7883 * unregister_netdevice(y1);
7884 * unregister_netdevice(y2);
7885 * ...
7886 * rtnl_unlock();
7887 * free_netdev(y1);
7888 * free_netdev(y2);
7889 *
7890 * We are invoked by rtnl_unlock().
7891 * This allows us to deal with problems:
7892 * 1) We can delete sysfs objects which invoke hotplug
7893 * without deadlocking with linkwatch via keventd.
7894 * 2) Since we run with the RTNL semaphore not held, we can sleep
7895 * safely in order to wait for the netdev refcnt to drop to zero.
7896 *
7897 * We must not return until all unregister events added during
7898 * the interval the lock was held have been completed.
7899 */
netdev_run_todo(void)7900 void netdev_run_todo(void)
7901 {
7902 struct list_head list;
7903
7904 /* Snapshot list, allow later requests */
7905 list_replace_init(&net_todo_list, &list);
7906
7907 __rtnl_unlock();
7908
7909
7910 /* Wait for rcu callbacks to finish before next phase */
7911 if (!list_empty(&list))
7912 rcu_barrier();
7913
7914 while (!list_empty(&list)) {
7915 struct net_device *dev
7916 = list_first_entry(&list, struct net_device, todo_list);
7917 list_del(&dev->todo_list);
7918
7919 rtnl_lock();
7920 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7921 __rtnl_unlock();
7922
7923 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7924 pr_err("network todo '%s' but state %d\n",
7925 dev->name, dev->reg_state);
7926 dump_stack();
7927 continue;
7928 }
7929
7930 dev->reg_state = NETREG_UNREGISTERED;
7931
7932 netdev_wait_allrefs(dev);
7933
7934 /* paranoia */
7935 BUG_ON(netdev_refcnt_read(dev));
7936 BUG_ON(!list_empty(&dev->ptype_all));
7937 BUG_ON(!list_empty(&dev->ptype_specific));
7938 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7939 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7940 WARN_ON(dev->dn_ptr);
7941
7942 if (dev->priv_destructor)
7943 dev->priv_destructor(dev);
7944 if (dev->needs_free_netdev)
7945 free_netdev(dev);
7946
7947 /* Report a network device has been unregistered */
7948 rtnl_lock();
7949 dev_net(dev)->dev_unreg_count--;
7950 __rtnl_unlock();
7951 wake_up(&netdev_unregistering_wq);
7952
7953 /* Free network device */
7954 kobject_put(&dev->dev.kobj);
7955 }
7956 }
7957
7958 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7959 * all the same fields in the same order as net_device_stats, with only
7960 * the type differing, but rtnl_link_stats64 may have additional fields
7961 * at the end for newer counters.
7962 */
netdev_stats_to_stats64(struct rtnl_link_stats64 * stats64,const struct net_device_stats * netdev_stats)7963 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7964 const struct net_device_stats *netdev_stats)
7965 {
7966 #if BITS_PER_LONG == 64
7967 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
7968 memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
7969 /* zero out counters that only exist in rtnl_link_stats64 */
7970 memset((char *)stats64 + sizeof(*netdev_stats), 0,
7971 sizeof(*stats64) - sizeof(*netdev_stats));
7972 #else
7973 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
7974 const unsigned long *src = (const unsigned long *)netdev_stats;
7975 u64 *dst = (u64 *)stats64;
7976
7977 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
7978 for (i = 0; i < n; i++)
7979 dst[i] = src[i];
7980 /* zero out counters that only exist in rtnl_link_stats64 */
7981 memset((char *)stats64 + n * sizeof(u64), 0,
7982 sizeof(*stats64) - n * sizeof(u64));
7983 #endif
7984 }
7985 EXPORT_SYMBOL(netdev_stats_to_stats64);
7986
7987 /**
7988 * dev_get_stats - get network device statistics
7989 * @dev: device to get statistics from
7990 * @storage: place to store stats
7991 *
7992 * Get network statistics from device. Return @storage.
7993 * The device driver may provide its own method by setting
7994 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7995 * otherwise the internal statistics structure is used.
7996 */
dev_get_stats(struct net_device * dev,struct rtnl_link_stats64 * storage)7997 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7998 struct rtnl_link_stats64 *storage)
7999 {
8000 const struct net_device_ops *ops = dev->netdev_ops;
8001
8002 if (ops->ndo_get_stats64) {
8003 memset(storage, 0, sizeof(*storage));
8004 ops->ndo_get_stats64(dev, storage);
8005 } else if (ops->ndo_get_stats) {
8006 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
8007 } else {
8008 netdev_stats_to_stats64(storage, &dev->stats);
8009 }
8010 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
8011 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
8012 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
8013 return storage;
8014 }
8015 EXPORT_SYMBOL(dev_get_stats);
8016
dev_ingress_queue_create(struct net_device * dev)8017 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
8018 {
8019 struct netdev_queue *queue = dev_ingress_queue(dev);
8020
8021 #ifdef CONFIG_NET_CLS_ACT
8022 if (queue)
8023 return queue;
8024 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
8025 if (!queue)
8026 return NULL;
8027 netdev_init_one_queue(dev, queue, NULL);
8028 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
8029 queue->qdisc_sleeping = &noop_qdisc;
8030 rcu_assign_pointer(dev->ingress_queue, queue);
8031 #endif
8032 return queue;
8033 }
8034
8035 static const struct ethtool_ops default_ethtool_ops;
8036
netdev_set_default_ethtool_ops(struct net_device * dev,const struct ethtool_ops * ops)8037 void netdev_set_default_ethtool_ops(struct net_device *dev,
8038 const struct ethtool_ops *ops)
8039 {
8040 if (dev->ethtool_ops == &default_ethtool_ops)
8041 dev->ethtool_ops = ops;
8042 }
8043 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
8044
netdev_freemem(struct net_device * dev)8045 void netdev_freemem(struct net_device *dev)
8046 {
8047 char *addr = (char *)dev - dev->padded;
8048
8049 kvfree(addr);
8050 }
8051
8052 /**
8053 * alloc_netdev_mqs - allocate network device
8054 * @sizeof_priv: size of private data to allocate space for
8055 * @name: device name format string
8056 * @name_assign_type: origin of device name
8057 * @setup: callback to initialize device
8058 * @txqs: the number of TX subqueues to allocate
8059 * @rxqs: the number of RX subqueues to allocate
8060 *
8061 * Allocates a struct net_device with private data area for driver use
8062 * and performs basic initialization. Also allocates subqueue structs
8063 * for each queue on the device.
8064 */
alloc_netdev_mqs(int sizeof_priv,const char * name,unsigned char name_assign_type,void (* setup)(struct net_device *),unsigned int txqs,unsigned int rxqs)8065 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
8066 unsigned char name_assign_type,
8067 void (*setup)(struct net_device *),
8068 unsigned int txqs, unsigned int rxqs)
8069 {
8070 struct net_device *dev;
8071 size_t alloc_size;
8072 struct net_device *p;
8073
8074 BUG_ON(strlen(name) >= sizeof(dev->name));
8075
8076 if (txqs < 1) {
8077 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
8078 return NULL;
8079 }
8080
8081 #ifdef CONFIG_SYSFS
8082 if (rxqs < 1) {
8083 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
8084 return NULL;
8085 }
8086 #endif
8087
8088 alloc_size = sizeof(struct net_device);
8089 if (sizeof_priv) {
8090 /* ensure 32-byte alignment of private area */
8091 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
8092 alloc_size += sizeof_priv;
8093 }
8094 /* ensure 32-byte alignment of whole construct */
8095 alloc_size += NETDEV_ALIGN - 1;
8096
8097 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
8098 if (!p)
8099 return NULL;
8100
8101 dev = PTR_ALIGN(p, NETDEV_ALIGN);
8102 dev->padded = (char *)dev - (char *)p;
8103
8104 dev->pcpu_refcnt = alloc_percpu(int);
8105 if (!dev->pcpu_refcnt)
8106 goto free_dev;
8107
8108 if (dev_addr_init(dev))
8109 goto free_pcpu;
8110
8111 dev_mc_init(dev);
8112 dev_uc_init(dev);
8113
8114 dev_net_set(dev, &init_net);
8115
8116 dev->gso_max_size = GSO_MAX_SIZE;
8117 dev->gso_max_segs = GSO_MAX_SEGS;
8118
8119 INIT_LIST_HEAD(&dev->napi_list);
8120 INIT_LIST_HEAD(&dev->unreg_list);
8121 INIT_LIST_HEAD(&dev->close_list);
8122 INIT_LIST_HEAD(&dev->link_watch_list);
8123 INIT_LIST_HEAD(&dev->adj_list.upper);
8124 INIT_LIST_HEAD(&dev->adj_list.lower);
8125 INIT_LIST_HEAD(&dev->ptype_all);
8126 INIT_LIST_HEAD(&dev->ptype_specific);
8127 #ifdef CONFIG_NET_SCHED
8128 hash_init(dev->qdisc_hash);
8129 #endif
8130 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
8131 setup(dev);
8132
8133 if (!dev->tx_queue_len) {
8134 dev->priv_flags |= IFF_NO_QUEUE;
8135 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
8136 }
8137
8138 dev->num_tx_queues = txqs;
8139 dev->real_num_tx_queues = txqs;
8140 if (netif_alloc_netdev_queues(dev))
8141 goto free_all;
8142
8143 #ifdef CONFIG_SYSFS
8144 dev->num_rx_queues = rxqs;
8145 dev->real_num_rx_queues = rxqs;
8146 if (netif_alloc_rx_queues(dev))
8147 goto free_all;
8148 #endif
8149
8150 strcpy(dev->name, name);
8151 dev->name_assign_type = name_assign_type;
8152 dev->group = INIT_NETDEV_GROUP;
8153 if (!dev->ethtool_ops)
8154 dev->ethtool_ops = &default_ethtool_ops;
8155
8156 nf_hook_ingress_init(dev);
8157
8158 return dev;
8159
8160 free_all:
8161 free_netdev(dev);
8162 return NULL;
8163
8164 free_pcpu:
8165 free_percpu(dev->pcpu_refcnt);
8166 free_dev:
8167 netdev_freemem(dev);
8168 return NULL;
8169 }
8170 EXPORT_SYMBOL(alloc_netdev_mqs);
8171
8172 /**
8173 * free_netdev - free network device
8174 * @dev: device
8175 *
8176 * This function does the last stage of destroying an allocated device
8177 * interface. The reference to the device object is released. If this
8178 * is the last reference then it will be freed.Must be called in process
8179 * context.
8180 */
free_netdev(struct net_device * dev)8181 void free_netdev(struct net_device *dev)
8182 {
8183 struct napi_struct *p, *n;
8184 struct bpf_prog *prog;
8185
8186 might_sleep();
8187 netif_free_tx_queues(dev);
8188 #ifdef CONFIG_SYSFS
8189 kvfree(dev->_rx);
8190 #endif
8191
8192 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
8193
8194 /* Flush device addresses */
8195 dev_addr_flush(dev);
8196
8197 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
8198 netif_napi_del(p);
8199
8200 free_percpu(dev->pcpu_refcnt);
8201 dev->pcpu_refcnt = NULL;
8202
8203 prog = rcu_dereference_protected(dev->xdp_prog, 1);
8204 if (prog) {
8205 bpf_prog_put(prog);
8206 static_key_slow_dec(&generic_xdp_needed);
8207 }
8208
8209 /* Compatibility with error handling in drivers */
8210 if (dev->reg_state == NETREG_UNINITIALIZED) {
8211 netdev_freemem(dev);
8212 return;
8213 }
8214
8215 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
8216 dev->reg_state = NETREG_RELEASED;
8217
8218 /* will free via device release */
8219 put_device(&dev->dev);
8220 }
8221 EXPORT_SYMBOL(free_netdev);
8222
8223 /**
8224 * synchronize_net - Synchronize with packet receive processing
8225 *
8226 * Wait for packets currently being received to be done.
8227 * Does not block later packets from starting.
8228 */
synchronize_net(void)8229 void synchronize_net(void)
8230 {
8231 might_sleep();
8232 if (rtnl_is_locked())
8233 synchronize_rcu_expedited();
8234 else
8235 synchronize_rcu();
8236 }
8237 EXPORT_SYMBOL(synchronize_net);
8238
8239 /**
8240 * unregister_netdevice_queue - remove device from the kernel
8241 * @dev: device
8242 * @head: list
8243 *
8244 * This function shuts down a device interface and removes it
8245 * from the kernel tables.
8246 * If head not NULL, device is queued to be unregistered later.
8247 *
8248 * Callers must hold the rtnl semaphore. You may want
8249 * unregister_netdev() instead of this.
8250 */
8251
unregister_netdevice_queue(struct net_device * dev,struct list_head * head)8252 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
8253 {
8254 ASSERT_RTNL();
8255
8256 if (head) {
8257 list_move_tail(&dev->unreg_list, head);
8258 } else {
8259 rollback_registered(dev);
8260 /* Finish processing unregister after unlock */
8261 net_set_todo(dev);
8262 }
8263 }
8264 EXPORT_SYMBOL(unregister_netdevice_queue);
8265
8266 /**
8267 * unregister_netdevice_many - unregister many devices
8268 * @head: list of devices
8269 *
8270 * Note: As most callers use a stack allocated list_head,
8271 * we force a list_del() to make sure stack wont be corrupted later.
8272 */
unregister_netdevice_many(struct list_head * head)8273 void unregister_netdevice_many(struct list_head *head)
8274 {
8275 struct net_device *dev;
8276
8277 if (!list_empty(head)) {
8278 rollback_registered_many(head);
8279 list_for_each_entry(dev, head, unreg_list)
8280 net_set_todo(dev);
8281 list_del(head);
8282 }
8283 }
8284 EXPORT_SYMBOL(unregister_netdevice_many);
8285
8286 /**
8287 * unregister_netdev - remove device from the kernel
8288 * @dev: device
8289 *
8290 * This function shuts down a device interface and removes it
8291 * from the kernel tables.
8292 *
8293 * This is just a wrapper for unregister_netdevice that takes
8294 * the rtnl semaphore. In general you want to use this and not
8295 * unregister_netdevice.
8296 */
unregister_netdev(struct net_device * dev)8297 void unregister_netdev(struct net_device *dev)
8298 {
8299 rtnl_lock();
8300 unregister_netdevice(dev);
8301 rtnl_unlock();
8302 }
8303 EXPORT_SYMBOL(unregister_netdev);
8304
8305 /**
8306 * dev_change_net_namespace - move device to different nethost namespace
8307 * @dev: device
8308 * @net: network namespace
8309 * @pat: If not NULL name pattern to try if the current device name
8310 * is already taken in the destination network namespace.
8311 *
8312 * This function shuts down a device interface and moves it
8313 * to a new network namespace. On success 0 is returned, on
8314 * a failure a netagive errno code is returned.
8315 *
8316 * Callers must hold the rtnl semaphore.
8317 */
8318
dev_change_net_namespace(struct net_device * dev,struct net * net,const char * pat)8319 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
8320 {
8321 int err;
8322
8323 ASSERT_RTNL();
8324
8325 /* Don't allow namespace local devices to be moved. */
8326 err = -EINVAL;
8327 if (dev->features & NETIF_F_NETNS_LOCAL)
8328 goto out;
8329
8330 /* Ensure the device has been registrered */
8331 if (dev->reg_state != NETREG_REGISTERED)
8332 goto out;
8333
8334 /* Get out if there is nothing todo */
8335 err = 0;
8336 if (net_eq(dev_net(dev), net))
8337 goto out;
8338
8339 /* Pick the destination device name, and ensure
8340 * we can use it in the destination network namespace.
8341 */
8342 err = -EEXIST;
8343 if (__dev_get_by_name(net, dev->name)) {
8344 /* We get here if we can't use the current device name */
8345 if (!pat)
8346 goto out;
8347 err = dev_get_valid_name(net, dev, pat);
8348 if (err < 0)
8349 goto out;
8350 }
8351
8352 /*
8353 * And now a mini version of register_netdevice unregister_netdevice.
8354 */
8355
8356 /* If device is running close it first. */
8357 dev_close(dev);
8358
8359 /* And unlink it from device chain */
8360 unlist_netdevice(dev);
8361
8362 synchronize_net();
8363
8364 /* Shutdown queueing discipline. */
8365 dev_shutdown(dev);
8366
8367 /* Notify protocols, that we are about to destroy
8368 * this device. They should clean all the things.
8369 *
8370 * Note that dev->reg_state stays at NETREG_REGISTERED.
8371 * This is wanted because this way 8021q and macvlan know
8372 * the device is just moving and can keep their slaves up.
8373 */
8374 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8375 rcu_barrier();
8376 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
8377 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
8378
8379 /*
8380 * Flush the unicast and multicast chains
8381 */
8382 dev_uc_flush(dev);
8383 dev_mc_flush(dev);
8384
8385 /* Send a netdev-removed uevent to the old namespace */
8386 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
8387 netdev_adjacent_del_links(dev);
8388
8389 /* Actually switch the network namespace */
8390 dev_net_set(dev, net);
8391
8392 /* If there is an ifindex conflict assign a new one */
8393 if (__dev_get_by_index(net, dev->ifindex))
8394 dev->ifindex = dev_new_index(net);
8395
8396 /* Send a netdev-add uevent to the new namespace */
8397 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
8398 netdev_adjacent_add_links(dev);
8399
8400 /* Fixup kobjects */
8401 err = device_rename(&dev->dev, dev->name);
8402 WARN_ON(err);
8403
8404 /* Add the device back in the hashes */
8405 list_netdevice(dev);
8406
8407 /* Notify protocols, that a new device appeared. */
8408 call_netdevice_notifiers(NETDEV_REGISTER, dev);
8409
8410 /*
8411 * Prevent userspace races by waiting until the network
8412 * device is fully setup before sending notifications.
8413 */
8414 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
8415
8416 synchronize_net();
8417 err = 0;
8418 out:
8419 return err;
8420 }
8421 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
8422
dev_cpu_dead(unsigned int oldcpu)8423 static int dev_cpu_dead(unsigned int oldcpu)
8424 {
8425 struct sk_buff **list_skb;
8426 struct sk_buff *skb;
8427 unsigned int cpu;
8428 struct softnet_data *sd, *oldsd, *remsd = NULL;
8429
8430 local_irq_disable();
8431 cpu = smp_processor_id();
8432 sd = &per_cpu(softnet_data, cpu);
8433 oldsd = &per_cpu(softnet_data, oldcpu);
8434
8435 /* Find end of our completion_queue. */
8436 list_skb = &sd->completion_queue;
8437 while (*list_skb)
8438 list_skb = &(*list_skb)->next;
8439 /* Append completion queue from offline CPU. */
8440 *list_skb = oldsd->completion_queue;
8441 oldsd->completion_queue = NULL;
8442
8443 /* Append output queue from offline CPU. */
8444 if (oldsd->output_queue) {
8445 *sd->output_queue_tailp = oldsd->output_queue;
8446 sd->output_queue_tailp = oldsd->output_queue_tailp;
8447 oldsd->output_queue = NULL;
8448 oldsd->output_queue_tailp = &oldsd->output_queue;
8449 }
8450 /* Append NAPI poll list from offline CPU, with one exception :
8451 * process_backlog() must be called by cpu owning percpu backlog.
8452 * We properly handle process_queue & input_pkt_queue later.
8453 */
8454 while (!list_empty(&oldsd->poll_list)) {
8455 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8456 struct napi_struct,
8457 poll_list);
8458
8459 list_del_init(&napi->poll_list);
8460 if (napi->poll == process_backlog)
8461 napi->state = 0;
8462 else
8463 ____napi_schedule(sd, napi);
8464 }
8465
8466 raise_softirq_irqoff(NET_TX_SOFTIRQ);
8467 local_irq_enable();
8468
8469 #ifdef CONFIG_RPS
8470 remsd = oldsd->rps_ipi_list;
8471 oldsd->rps_ipi_list = NULL;
8472 #endif
8473 /* send out pending IPI's on offline CPU */
8474 net_rps_send_ipi(remsd);
8475
8476 /* Process offline CPU's input_pkt_queue */
8477 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
8478 netif_rx_ni(skb);
8479 input_queue_head_incr(oldsd);
8480 }
8481 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
8482 netif_rx_ni(skb);
8483 input_queue_head_incr(oldsd);
8484 }
8485
8486 return 0;
8487 }
8488
8489 /**
8490 * netdev_increment_features - increment feature set by one
8491 * @all: current feature set
8492 * @one: new feature set
8493 * @mask: mask feature set
8494 *
8495 * Computes a new feature set after adding a device with feature set
8496 * @one to the master device with current feature set @all. Will not
8497 * enable anything that is off in @mask. Returns the new feature set.
8498 */
netdev_increment_features(netdev_features_t all,netdev_features_t one,netdev_features_t mask)8499 netdev_features_t netdev_increment_features(netdev_features_t all,
8500 netdev_features_t one, netdev_features_t mask)
8501 {
8502 if (mask & NETIF_F_HW_CSUM)
8503 mask |= NETIF_F_CSUM_MASK;
8504 mask |= NETIF_F_VLAN_CHALLENGED;
8505
8506 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
8507 all &= one | ~NETIF_F_ALL_FOR_ALL;
8508
8509 /* If one device supports hw checksumming, set for all. */
8510 if (all & NETIF_F_HW_CSUM)
8511 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
8512
8513 return all;
8514 }
8515 EXPORT_SYMBOL(netdev_increment_features);
8516
netdev_create_hash(void)8517 static struct hlist_head * __net_init netdev_create_hash(void)
8518 {
8519 int i;
8520 struct hlist_head *hash;
8521
8522 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8523 if (hash != NULL)
8524 for (i = 0; i < NETDEV_HASHENTRIES; i++)
8525 INIT_HLIST_HEAD(&hash[i]);
8526
8527 return hash;
8528 }
8529
8530 /* Initialize per network namespace state */
netdev_init(struct net * net)8531 static int __net_init netdev_init(struct net *net)
8532 {
8533 if (net != &init_net)
8534 INIT_LIST_HEAD(&net->dev_base_head);
8535
8536 net->dev_name_head = netdev_create_hash();
8537 if (net->dev_name_head == NULL)
8538 goto err_name;
8539
8540 net->dev_index_head = netdev_create_hash();
8541 if (net->dev_index_head == NULL)
8542 goto err_idx;
8543
8544 return 0;
8545
8546 err_idx:
8547 kfree(net->dev_name_head);
8548 err_name:
8549 return -ENOMEM;
8550 }
8551
8552 /**
8553 * netdev_drivername - network driver for the device
8554 * @dev: network device
8555 *
8556 * Determine network driver for device.
8557 */
netdev_drivername(const struct net_device * dev)8558 const char *netdev_drivername(const struct net_device *dev)
8559 {
8560 const struct device_driver *driver;
8561 const struct device *parent;
8562 const char *empty = "";
8563
8564 parent = dev->dev.parent;
8565 if (!parent)
8566 return empty;
8567
8568 driver = parent->driver;
8569 if (driver && driver->name)
8570 return driver->name;
8571 return empty;
8572 }
8573
__netdev_printk(const char * level,const struct net_device * dev,struct va_format * vaf)8574 static void __netdev_printk(const char *level, const struct net_device *dev,
8575 struct va_format *vaf)
8576 {
8577 if (dev && dev->dev.parent) {
8578 dev_printk_emit(level[1] - '0',
8579 dev->dev.parent,
8580 "%s %s %s%s: %pV",
8581 dev_driver_string(dev->dev.parent),
8582 dev_name(dev->dev.parent),
8583 netdev_name(dev), netdev_reg_state(dev),
8584 vaf);
8585 } else if (dev) {
8586 printk("%s%s%s: %pV",
8587 level, netdev_name(dev), netdev_reg_state(dev), vaf);
8588 } else {
8589 printk("%s(NULL net_device): %pV", level, vaf);
8590 }
8591 }
8592
netdev_printk(const char * level,const struct net_device * dev,const char * format,...)8593 void netdev_printk(const char *level, const struct net_device *dev,
8594 const char *format, ...)
8595 {
8596 struct va_format vaf;
8597 va_list args;
8598
8599 va_start(args, format);
8600
8601 vaf.fmt = format;
8602 vaf.va = &args;
8603
8604 __netdev_printk(level, dev, &vaf);
8605
8606 va_end(args);
8607 }
8608 EXPORT_SYMBOL(netdev_printk);
8609
8610 #define define_netdev_printk_level(func, level) \
8611 void func(const struct net_device *dev, const char *fmt, ...) \
8612 { \
8613 struct va_format vaf; \
8614 va_list args; \
8615 \
8616 va_start(args, fmt); \
8617 \
8618 vaf.fmt = fmt; \
8619 vaf.va = &args; \
8620 \
8621 __netdev_printk(level, dev, &vaf); \
8622 \
8623 va_end(args); \
8624 } \
8625 EXPORT_SYMBOL(func);
8626
8627 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8628 define_netdev_printk_level(netdev_alert, KERN_ALERT);
8629 define_netdev_printk_level(netdev_crit, KERN_CRIT);
8630 define_netdev_printk_level(netdev_err, KERN_ERR);
8631 define_netdev_printk_level(netdev_warn, KERN_WARNING);
8632 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8633 define_netdev_printk_level(netdev_info, KERN_INFO);
8634
netdev_exit(struct net * net)8635 static void __net_exit netdev_exit(struct net *net)
8636 {
8637 kfree(net->dev_name_head);
8638 kfree(net->dev_index_head);
8639 }
8640
8641 static struct pernet_operations __net_initdata netdev_net_ops = {
8642 .init = netdev_init,
8643 .exit = netdev_exit,
8644 };
8645
default_device_exit(struct net * net)8646 static void __net_exit default_device_exit(struct net *net)
8647 {
8648 struct net_device *dev, *aux;
8649 /*
8650 * Push all migratable network devices back to the
8651 * initial network namespace
8652 */
8653 rtnl_lock();
8654 for_each_netdev_safe(net, dev, aux) {
8655 int err;
8656 char fb_name[IFNAMSIZ];
8657
8658 /* Ignore unmoveable devices (i.e. loopback) */
8659 if (dev->features & NETIF_F_NETNS_LOCAL)
8660 continue;
8661
8662 /* Leave virtual devices for the generic cleanup */
8663 if (dev->rtnl_link_ops)
8664 continue;
8665
8666 /* Push remaining network devices to init_net */
8667 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8668 if (__dev_get_by_name(&init_net, fb_name))
8669 snprintf(fb_name, IFNAMSIZ, "dev%%d");
8670 err = dev_change_net_namespace(dev, &init_net, fb_name);
8671 if (err) {
8672 pr_emerg("%s: failed to move %s to init_net: %d\n",
8673 __func__, dev->name, err);
8674 BUG();
8675 }
8676 }
8677 rtnl_unlock();
8678 }
8679
rtnl_lock_unregistering(struct list_head * net_list)8680 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8681 {
8682 /* Return with the rtnl_lock held when there are no network
8683 * devices unregistering in any network namespace in net_list.
8684 */
8685 struct net *net;
8686 bool unregistering;
8687 DEFINE_WAIT_FUNC(wait, woken_wake_function);
8688
8689 add_wait_queue(&netdev_unregistering_wq, &wait);
8690 for (;;) {
8691 unregistering = false;
8692 rtnl_lock();
8693 list_for_each_entry(net, net_list, exit_list) {
8694 if (net->dev_unreg_count > 0) {
8695 unregistering = true;
8696 break;
8697 }
8698 }
8699 if (!unregistering)
8700 break;
8701 __rtnl_unlock();
8702
8703 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8704 }
8705 remove_wait_queue(&netdev_unregistering_wq, &wait);
8706 }
8707
default_device_exit_batch(struct list_head * net_list)8708 static void __net_exit default_device_exit_batch(struct list_head *net_list)
8709 {
8710 /* At exit all network devices most be removed from a network
8711 * namespace. Do this in the reverse order of registration.
8712 * Do this across as many network namespaces as possible to
8713 * improve batching efficiency.
8714 */
8715 struct net_device *dev;
8716 struct net *net;
8717 LIST_HEAD(dev_kill_list);
8718
8719 /* To prevent network device cleanup code from dereferencing
8720 * loopback devices or network devices that have been freed
8721 * wait here for all pending unregistrations to complete,
8722 * before unregistring the loopback device and allowing the
8723 * network namespace be freed.
8724 *
8725 * The netdev todo list containing all network devices
8726 * unregistrations that happen in default_device_exit_batch
8727 * will run in the rtnl_unlock() at the end of
8728 * default_device_exit_batch.
8729 */
8730 rtnl_lock_unregistering(net_list);
8731 list_for_each_entry(net, net_list, exit_list) {
8732 for_each_netdev_reverse(net, dev) {
8733 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8734 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8735 else
8736 unregister_netdevice_queue(dev, &dev_kill_list);
8737 }
8738 }
8739 unregister_netdevice_many(&dev_kill_list);
8740 rtnl_unlock();
8741 }
8742
8743 static struct pernet_operations __net_initdata default_device_ops = {
8744 .exit = default_device_exit,
8745 .exit_batch = default_device_exit_batch,
8746 };
8747
8748 /*
8749 * Initialize the DEV module. At boot time this walks the device list and
8750 * unhooks any devices that fail to initialise (normally hardware not
8751 * present) and leaves us with a valid list of present and active devices.
8752 *
8753 */
8754
8755 /*
8756 * This is called single threaded during boot, so no need
8757 * to take the rtnl semaphore.
8758 */
net_dev_init(void)8759 static int __init net_dev_init(void)
8760 {
8761 int i, rc = -ENOMEM;
8762
8763 BUG_ON(!dev_boot_phase);
8764
8765 if (dev_proc_init())
8766 goto out;
8767
8768 if (netdev_kobject_init())
8769 goto out;
8770
8771 INIT_LIST_HEAD(&ptype_all);
8772 for (i = 0; i < PTYPE_HASH_SIZE; i++)
8773 INIT_LIST_HEAD(&ptype_base[i]);
8774
8775 INIT_LIST_HEAD(&offload_base);
8776
8777 if (register_pernet_subsys(&netdev_net_ops))
8778 goto out;
8779
8780 /*
8781 * Initialise the packet receive queues.
8782 */
8783
8784 for_each_possible_cpu(i) {
8785 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
8786 struct softnet_data *sd = &per_cpu(softnet_data, i);
8787
8788 INIT_WORK(flush, flush_backlog);
8789
8790 skb_queue_head_init(&sd->input_pkt_queue);
8791 skb_queue_head_init(&sd->process_queue);
8792 INIT_LIST_HEAD(&sd->poll_list);
8793 sd->output_queue_tailp = &sd->output_queue;
8794 #ifdef CONFIG_RPS
8795 sd->csd.func = rps_trigger_softirq;
8796 sd->csd.info = sd;
8797 sd->cpu = i;
8798 #endif
8799
8800 sd->backlog.poll = process_backlog;
8801 sd->backlog.weight = weight_p;
8802 }
8803
8804 dev_boot_phase = 0;
8805
8806 /* The loopback device is special if any other network devices
8807 * is present in a network namespace the loopback device must
8808 * be present. Since we now dynamically allocate and free the
8809 * loopback device ensure this invariant is maintained by
8810 * keeping the loopback device as the first device on the
8811 * list of network devices. Ensuring the loopback devices
8812 * is the first device that appears and the last network device
8813 * that disappears.
8814 */
8815 if (register_pernet_device(&loopback_net_ops))
8816 goto out;
8817
8818 if (register_pernet_device(&default_device_ops))
8819 goto out;
8820
8821 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8822 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8823
8824 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8825 NULL, dev_cpu_dead);
8826 WARN_ON(rc < 0);
8827 rc = 0;
8828 out:
8829 return rc;
8830 }
8831
8832 subsys_initcall(net_dev_init);
8833