• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  /* SPDX-License-Identifier: GPL-2.0-or-later */
2  /*
3   * INET		An implementation of the TCP/IP protocol suite for the LINUX
4   *		operating system.  INET is implemented using the  BSD Socket
5   *		interface as the means of communication with the user level.
6   *
7   *		Definitions for the Interfaces handler.
8   *
9   * Version:	@(#)dev.h	1.0.10	08/12/93
10   *
11   * Authors:	Ross Biro
12   *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13   *		Corey Minyard <wf-rch!minyard@relay.EU.net>
14   *		Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
15   *		Alan Cox, <alan@lxorguk.ukuu.org.uk>
16   *		Bjorn Ekwall. <bj0rn@blox.se>
17   *              Pekka Riikonen <priikone@poseidon.pspt.fi>
18   *
19   *		Moved to /usr/include/linux for NET3
20   */
21  #ifndef _LINUX_NETDEVICE_H
22  #define _LINUX_NETDEVICE_H
23  
24  #include <linux/timer.h>
25  #include <linux/bug.h>
26  #include <linux/delay.h>
27  #include <linux/atomic.h>
28  #include <linux/prefetch.h>
29  #include <asm/cache.h>
30  #include <asm/byteorder.h>
31  #include <asm/local.h>
32  
33  #include <linux/percpu.h>
34  #include <linux/rculist.h>
35  #include <linux/workqueue.h>
36  #include <linux/dynamic_queue_limits.h>
37  
38  #include <net/net_namespace.h>
39  #ifdef CONFIG_DCB
40  #include <net/dcbnl.h>
41  #endif
42  #include <net/netprio_cgroup.h>
43  #include <net/xdp.h>
44  
45  #include <linux/netdev_features.h>
46  #include <linux/neighbour.h>
47  #include <uapi/linux/netdevice.h>
48  #include <uapi/linux/if_bonding.h>
49  #include <uapi/linux/pkt_cls.h>
50  #include <linux/hashtable.h>
51  #include <linux/rbtree.h>
52  #include <net/net_trackers.h>
53  #include <net/net_debug.h>
54  #include <linux/android_kabi.h>
55  
56  struct netpoll_info;
57  struct device;
58  struct ethtool_ops;
59  struct phy_device;
60  struct dsa_port;
61  struct ip_tunnel_parm;
62  struct macsec_context;
63  struct macsec_ops;
64  struct netdev_name_node;
65  struct sd_flow_limit;
66  struct sfp_bus;
67  /* 802.11 specific */
68  struct wireless_dev;
69  /* 802.15.4 specific */
70  struct wpan_dev;
71  struct mpls_dev;
72  /* UDP Tunnel offloads */
73  struct udp_tunnel_info;
74  struct udp_tunnel_nic_info;
75  struct udp_tunnel_nic;
76  struct bpf_prog;
77  struct xdp_buff;
78  
79  void synchronize_net(void);
80  void netdev_set_default_ethtool_ops(struct net_device *dev,
81  				    const struct ethtool_ops *ops);
82  
83  /* Backlog congestion levels */
84  #define NET_RX_SUCCESS		0	/* keep 'em coming, baby */
85  #define NET_RX_DROP		1	/* packet dropped */
86  
87  #define MAX_NEST_DEV 8
88  
89  /*
90   * Transmit return codes: transmit return codes originate from three different
91   * namespaces:
92   *
93   * - qdisc return codes
94   * - driver transmit return codes
95   * - errno values
96   *
97   * Drivers are allowed to return any one of those in their hard_start_xmit()
98   * function. Real network devices commonly used with qdiscs should only return
99   * the driver transmit return codes though - when qdiscs are used, the actual
100   * transmission happens asynchronously, so the value is not propagated to
101   * higher layers. Virtual network devices transmit synchronously; in this case
102   * the driver transmit return codes are consumed by dev_queue_xmit(), and all
103   * others are propagated to higher layers.
104   */
105  
106  /* qdisc ->enqueue() return codes. */
107  #define NET_XMIT_SUCCESS	0x00
108  #define NET_XMIT_DROP		0x01	/* skb dropped			*/
109  #define NET_XMIT_CN		0x02	/* congestion notification	*/
110  #define NET_XMIT_MASK		0x0f	/* qdisc flags in net/sch_generic.h */
111  
112  /* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
113   * indicates that the device will soon be dropping packets, or already drops
114   * some packets of the same priority; prompting us to send less aggressively. */
115  #define net_xmit_eval(e)	((e) == NET_XMIT_CN ? 0 : (e))
116  #define net_xmit_errno(e)	((e) != NET_XMIT_CN ? -ENOBUFS : 0)
117  
118  /* Driver transmit return codes */
119  #define NETDEV_TX_MASK		0xf0
120  
121  enum netdev_tx {
122  	__NETDEV_TX_MIN	 = INT_MIN,	/* make sure enum is signed */
123  	NETDEV_TX_OK	 = 0x00,	/* driver took care of packet */
124  	NETDEV_TX_BUSY	 = 0x10,	/* driver tx path was busy*/
125  };
126  typedef enum netdev_tx netdev_tx_t;
127  
128  /*
129   * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
130   * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
131   */
dev_xmit_complete(int rc)132  static inline bool dev_xmit_complete(int rc)
133  {
134  	/*
135  	 * Positive cases with an skb consumed by a driver:
136  	 * - successful transmission (rc == NETDEV_TX_OK)
137  	 * - error while transmitting (rc < 0)
138  	 * - error while queueing to a different device (rc & NET_XMIT_MASK)
139  	 */
140  	if (likely(rc < NET_XMIT_MASK))
141  		return true;
142  
143  	return false;
144  }
145  
146  /*
147   *	Compute the worst-case header length according to the protocols
148   *	used.
149   */
150  
151  #if defined(CONFIG_HYPERV_NET)
152  # define LL_MAX_HEADER 128
153  #elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
154  # if defined(CONFIG_MAC80211_MESH)
155  #  define LL_MAX_HEADER 128
156  # else
157  #  define LL_MAX_HEADER 96
158  # endif
159  #else
160  # define LL_MAX_HEADER 32
161  #endif
162  
163  #if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
164      !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
165  #define MAX_HEADER LL_MAX_HEADER
166  #else
167  #define MAX_HEADER (LL_MAX_HEADER + 48)
168  #endif
169  
170  /*
171   *	Old network device statistics. Fields are native words
172   *	(unsigned long) so they can be read and written atomically.
173   */
174  
175  #define NET_DEV_STAT(FIELD)			\
176  	union {					\
177  		unsigned long FIELD;		\
178  		atomic_long_t __##FIELD;	\
179  	}
180  
181  struct net_device_stats {
182  	NET_DEV_STAT(rx_packets);
183  	NET_DEV_STAT(tx_packets);
184  	NET_DEV_STAT(rx_bytes);
185  	NET_DEV_STAT(tx_bytes);
186  	NET_DEV_STAT(rx_errors);
187  	NET_DEV_STAT(tx_errors);
188  	NET_DEV_STAT(rx_dropped);
189  	NET_DEV_STAT(tx_dropped);
190  	NET_DEV_STAT(multicast);
191  	NET_DEV_STAT(collisions);
192  	NET_DEV_STAT(rx_length_errors);
193  	NET_DEV_STAT(rx_over_errors);
194  	NET_DEV_STAT(rx_crc_errors);
195  	NET_DEV_STAT(rx_frame_errors);
196  	NET_DEV_STAT(rx_fifo_errors);
197  	NET_DEV_STAT(rx_missed_errors);
198  	NET_DEV_STAT(tx_aborted_errors);
199  	NET_DEV_STAT(tx_carrier_errors);
200  	NET_DEV_STAT(tx_fifo_errors);
201  	NET_DEV_STAT(tx_heartbeat_errors);
202  	NET_DEV_STAT(tx_window_errors);
203  	NET_DEV_STAT(rx_compressed);
204  	NET_DEV_STAT(tx_compressed);
205  };
206  #undef NET_DEV_STAT
207  
208  /* per-cpu stats, allocated on demand.
209   * Try to fit them in a single cache line, for dev_get_stats() sake.
210   */
211  struct net_device_core_stats {
212  	unsigned long	rx_dropped;
213  	unsigned long	tx_dropped;
214  	unsigned long	rx_nohandler;
215  	unsigned long	rx_otherhost_dropped;
216  } __aligned(4 * sizeof(unsigned long));
217  
218  #include <linux/cache.h>
219  #include <linux/skbuff.h>
220  
221  #ifdef CONFIG_RPS
222  #include <linux/static_key.h>
223  extern struct static_key_false rps_needed;
224  extern struct static_key_false rfs_needed;
225  #endif
226  
227  struct neighbour;
228  struct neigh_parms;
229  struct sk_buff;
230  
231  struct netdev_hw_addr {
232  	struct list_head	list;
233  	struct rb_node		node;
234  	unsigned char		addr[MAX_ADDR_LEN];
235  	unsigned char		type;
236  #define NETDEV_HW_ADDR_T_LAN		1
237  #define NETDEV_HW_ADDR_T_SAN		2
238  #define NETDEV_HW_ADDR_T_UNICAST	3
239  #define NETDEV_HW_ADDR_T_MULTICAST	4
240  	bool			global_use;
241  	int			sync_cnt;
242  	int			refcount;
243  	int			synced;
244  	struct rcu_head		rcu_head;
245  };
246  
247  struct netdev_hw_addr_list {
248  	struct list_head	list;
249  	int			count;
250  
251  	/* Auxiliary tree for faster lookup on addition and deletion */
252  	struct rb_root		tree;
253  };
254  
255  #define netdev_hw_addr_list_count(l) ((l)->count)
256  #define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
257  #define netdev_hw_addr_list_for_each(ha, l) \
258  	list_for_each_entry(ha, &(l)->list, list)
259  
260  #define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
261  #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
262  #define netdev_for_each_uc_addr(ha, dev) \
263  	netdev_hw_addr_list_for_each(ha, &(dev)->uc)
264  #define netdev_for_each_synced_uc_addr(_ha, _dev) \
265  	netdev_for_each_uc_addr((_ha), (_dev)) \
266  		if ((_ha)->sync_cnt)
267  
268  #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
269  #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
270  #define netdev_for_each_mc_addr(ha, dev) \
271  	netdev_hw_addr_list_for_each(ha, &(dev)->mc)
272  #define netdev_for_each_synced_mc_addr(_ha, _dev) \
273  	netdev_for_each_mc_addr((_ha), (_dev)) \
274  		if ((_ha)->sync_cnt)
275  
276  struct hh_cache {
277  	unsigned int	hh_len;
278  	seqlock_t	hh_lock;
279  
280  	/* cached hardware header; allow for machine alignment needs.        */
281  #define HH_DATA_MOD	16
282  #define HH_DATA_OFF(__len) \
283  	(HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
284  #define HH_DATA_ALIGN(__len) \
285  	(((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
286  	unsigned long	hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
287  };
288  
289  /* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
290   * Alternative is:
291   *   dev->hard_header_len ? (dev->hard_header_len +
292   *                           (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
293   *
294   * We could use other alignment values, but we must maintain the
295   * relationship HH alignment <= LL alignment.
296   */
297  #define LL_RESERVED_SPACE(dev) \
298  	((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \
299  	  & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
300  #define LL_RESERVED_SPACE_EXTRA(dev,extra) \
301  	((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \
302  	  & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
303  
304  struct header_ops {
305  	int	(*create) (struct sk_buff *skb, struct net_device *dev,
306  			   unsigned short type, const void *daddr,
307  			   const void *saddr, unsigned int len);
308  	int	(*parse)(const struct sk_buff *skb, unsigned char *haddr);
309  	int	(*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
310  	void	(*cache_update)(struct hh_cache *hh,
311  				const struct net_device *dev,
312  				const unsigned char *haddr);
313  	bool	(*validate)(const char *ll_header, unsigned int len);
314  	__be16	(*parse_protocol)(const struct sk_buff *skb);
315  
316  	ANDROID_KABI_RESERVE(1);
317  	ANDROID_KABI_RESERVE(2);
318  };
319  
320  /* These flag bits are private to the generic network queueing
321   * layer; they may not be explicitly referenced by any other
322   * code.
323   */
324  
325  enum netdev_state_t {
326  	__LINK_STATE_START,
327  	__LINK_STATE_PRESENT,
328  	__LINK_STATE_NOCARRIER,
329  	__LINK_STATE_LINKWATCH_PENDING,
330  	__LINK_STATE_DORMANT,
331  	__LINK_STATE_TESTING,
332  };
333  
334  struct gro_list {
335  	struct list_head	list;
336  	int			count;
337  };
338  
339  /*
340   * size of gro hash buckets, must less than bit number of
341   * napi_struct::gro_bitmask
342   */
343  #define GRO_HASH_BUCKETS	8
344  
345  /*
346   * Structure for NAPI scheduling similar to tasklet but with weighting
347   */
348  struct napi_struct {
349  	/* The poll_list must only be managed by the entity which
350  	 * changes the state of the NAPI_STATE_SCHED bit.  This means
351  	 * whoever atomically sets that bit can add this napi_struct
352  	 * to the per-CPU poll_list, and whoever clears that bit
353  	 * can remove from the list right before clearing the bit.
354  	 */
355  	struct list_head	poll_list;
356  
357  	unsigned long		state;
358  	int			weight;
359  	int			defer_hard_irqs_count;
360  	unsigned long		gro_bitmask;
361  	int			(*poll)(struct napi_struct *, int);
362  #ifdef CONFIG_NETPOLL
363  	int			poll_owner;
364  #endif
365  	struct net_device	*dev;
366  	struct gro_list		gro_hash[GRO_HASH_BUCKETS];
367  	struct sk_buff		*skb;
368  	struct list_head	rx_list; /* Pending GRO_NORMAL skbs */
369  	int			rx_count; /* length of rx_list */
370  	struct hrtimer		timer;
371  	struct list_head	dev_list;
372  	struct hlist_node	napi_hash_node;
373  	unsigned int		napi_id;
374  	struct task_struct	*thread;
375  
376  	ANDROID_KABI_RESERVE(1);
377  	ANDROID_KABI_RESERVE(2);
378  	ANDROID_KABI_RESERVE(3);
379  	ANDROID_KABI_RESERVE(4);
380  };
381  
382  enum {
383  	NAPI_STATE_SCHED,		/* Poll is scheduled */
384  	NAPI_STATE_MISSED,		/* reschedule a napi */
385  	NAPI_STATE_DISABLE,		/* Disable pending */
386  	NAPI_STATE_NPSVC,		/* Netpoll - don't dequeue from poll_list */
387  	NAPI_STATE_LISTED,		/* NAPI added to system lists */
388  	NAPI_STATE_NO_BUSY_POLL,	/* Do not add in napi_hash, no busy polling */
389  	NAPI_STATE_IN_BUSY_POLL,	/* sk_busy_loop() owns this NAPI */
390  	NAPI_STATE_PREFER_BUSY_POLL,	/* prefer busy-polling over softirq processing*/
391  	NAPI_STATE_THREADED,		/* The poll is performed inside its own thread*/
392  	NAPI_STATE_SCHED_THREADED,	/* Napi is currently scheduled in threaded mode */
393  };
394  
395  enum {
396  	NAPIF_STATE_SCHED		= BIT(NAPI_STATE_SCHED),
397  	NAPIF_STATE_MISSED		= BIT(NAPI_STATE_MISSED),
398  	NAPIF_STATE_DISABLE		= BIT(NAPI_STATE_DISABLE),
399  	NAPIF_STATE_NPSVC		= BIT(NAPI_STATE_NPSVC),
400  	NAPIF_STATE_LISTED		= BIT(NAPI_STATE_LISTED),
401  	NAPIF_STATE_NO_BUSY_POLL	= BIT(NAPI_STATE_NO_BUSY_POLL),
402  	NAPIF_STATE_IN_BUSY_POLL	= BIT(NAPI_STATE_IN_BUSY_POLL),
403  	NAPIF_STATE_PREFER_BUSY_POLL	= BIT(NAPI_STATE_PREFER_BUSY_POLL),
404  	NAPIF_STATE_THREADED		= BIT(NAPI_STATE_THREADED),
405  	NAPIF_STATE_SCHED_THREADED	= BIT(NAPI_STATE_SCHED_THREADED),
406  };
407  
408  enum gro_result {
409  	GRO_MERGED,
410  	GRO_MERGED_FREE,
411  	GRO_HELD,
412  	GRO_NORMAL,
413  	GRO_CONSUMED,
414  };
415  typedef enum gro_result gro_result_t;
416  
417  /*
418   * enum rx_handler_result - Possible return values for rx_handlers.
419   * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
420   * further.
421   * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
422   * case skb->dev was changed by rx_handler.
423   * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
424   * @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
425   *
426   * rx_handlers are functions called from inside __netif_receive_skb(), to do
427   * special processing of the skb, prior to delivery to protocol handlers.
428   *
429   * Currently, a net_device can only have a single rx_handler registered. Trying
430   * to register a second rx_handler will return -EBUSY.
431   *
432   * To register a rx_handler on a net_device, use netdev_rx_handler_register().
433   * To unregister a rx_handler on a net_device, use
434   * netdev_rx_handler_unregister().
435   *
436   * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
437   * do with the skb.
438   *
439   * If the rx_handler consumed the skb in some way, it should return
440   * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
441   * the skb to be delivered in some other way.
442   *
443   * If the rx_handler changed skb->dev, to divert the skb to another
444   * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
445   * new device will be called if it exists.
446   *
447   * If the rx_handler decides the skb should be ignored, it should return
448   * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
449   * are registered on exact device (ptype->dev == skb->dev).
450   *
451   * If the rx_handler didn't change skb->dev, but wants the skb to be normally
452   * delivered, it should return RX_HANDLER_PASS.
453   *
454   * A device without a registered rx_handler will behave as if rx_handler
455   * returned RX_HANDLER_PASS.
456   */
457  
458  enum rx_handler_result {
459  	RX_HANDLER_CONSUMED,
460  	RX_HANDLER_ANOTHER,
461  	RX_HANDLER_EXACT,
462  	RX_HANDLER_PASS,
463  };
464  typedef enum rx_handler_result rx_handler_result_t;
465  typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
466  
467  void __napi_schedule(struct napi_struct *n);
468  void __napi_schedule_irqoff(struct napi_struct *n);
469  
napi_disable_pending(struct napi_struct * n)470  static inline bool napi_disable_pending(struct napi_struct *n)
471  {
472  	return test_bit(NAPI_STATE_DISABLE, &n->state);
473  }
474  
napi_prefer_busy_poll(struct napi_struct * n)475  static inline bool napi_prefer_busy_poll(struct napi_struct *n)
476  {
477  	return test_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
478  }
479  
480  bool napi_schedule_prep(struct napi_struct *n);
481  
482  /**
483   *	napi_schedule - schedule NAPI poll
484   *	@n: NAPI context
485   *
486   * Schedule NAPI poll routine to be called if it is not already
487   * running.
488   */
napi_schedule(struct napi_struct * n)489  static inline void napi_schedule(struct napi_struct *n)
490  {
491  	if (napi_schedule_prep(n))
492  		__napi_schedule(n);
493  }
494  
495  /**
496   *	napi_schedule_irqoff - schedule NAPI poll
497   *	@n: NAPI context
498   *
499   * Variant of napi_schedule(), assuming hard irqs are masked.
500   */
napi_schedule_irqoff(struct napi_struct * n)501  static inline void napi_schedule_irqoff(struct napi_struct *n)
502  {
503  	if (napi_schedule_prep(n))
504  		__napi_schedule_irqoff(n);
505  }
506  
507  /* Try to reschedule poll. Called by dev->poll() after napi_complete().  */
napi_reschedule(struct napi_struct * napi)508  static inline bool napi_reschedule(struct napi_struct *napi)
509  {
510  	if (napi_schedule_prep(napi)) {
511  		__napi_schedule(napi);
512  		return true;
513  	}
514  	return false;
515  }
516  
517  bool napi_complete_done(struct napi_struct *n, int work_done);
518  /**
519   *	napi_complete - NAPI processing complete
520   *	@n: NAPI context
521   *
522   * Mark NAPI processing as complete.
523   * Consider using napi_complete_done() instead.
524   * Return false if device should avoid rearming interrupts.
525   */
napi_complete(struct napi_struct * n)526  static inline bool napi_complete(struct napi_struct *n)
527  {
528  	return napi_complete_done(n, 0);
529  }
530  
531  int dev_set_threaded(struct net_device *dev, bool threaded);
532  
533  /**
534   *	napi_disable - prevent NAPI from scheduling
535   *	@n: NAPI context
536   *
537   * Stop NAPI from being scheduled on this context.
538   * Waits till any outstanding processing completes.
539   */
540  void napi_disable(struct napi_struct *n);
541  
542  void napi_enable(struct napi_struct *n);
543  
544  /**
545   *	napi_synchronize - wait until NAPI is not running
546   *	@n: NAPI context
547   *
548   * Wait until NAPI is done being scheduled on this context.
549   * Waits till any outstanding processing completes but
550   * does not disable future activations.
551   */
napi_synchronize(const struct napi_struct * n)552  static inline void napi_synchronize(const struct napi_struct *n)
553  {
554  	if (IS_ENABLED(CONFIG_SMP))
555  		while (test_bit(NAPI_STATE_SCHED, &n->state))
556  			msleep(1);
557  	else
558  		barrier();
559  }
560  
561  /**
562   *	napi_if_scheduled_mark_missed - if napi is running, set the
563   *	NAPIF_STATE_MISSED
564   *	@n: NAPI context
565   *
566   * If napi is running, set the NAPIF_STATE_MISSED, and return true if
567   * NAPI is scheduled.
568   **/
napi_if_scheduled_mark_missed(struct napi_struct * n)569  static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
570  {
571  	unsigned long val, new;
572  
573  	val = READ_ONCE(n->state);
574  	do {
575  		if (val & NAPIF_STATE_DISABLE)
576  			return true;
577  
578  		if (!(val & NAPIF_STATE_SCHED))
579  			return false;
580  
581  		new = val | NAPIF_STATE_MISSED;
582  	} while (!try_cmpxchg(&n->state, &val, new));
583  
584  	return true;
585  }
586  
587  enum netdev_queue_state_t {
588  	__QUEUE_STATE_DRV_XOFF,
589  	__QUEUE_STATE_STACK_XOFF,
590  	__QUEUE_STATE_FROZEN,
591  };
592  
593  #define QUEUE_STATE_DRV_XOFF	(1 << __QUEUE_STATE_DRV_XOFF)
594  #define QUEUE_STATE_STACK_XOFF	(1 << __QUEUE_STATE_STACK_XOFF)
595  #define QUEUE_STATE_FROZEN	(1 << __QUEUE_STATE_FROZEN)
596  
597  #define QUEUE_STATE_ANY_XOFF	(QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
598  #define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
599  					QUEUE_STATE_FROZEN)
600  #define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
601  					QUEUE_STATE_FROZEN)
602  
603  /*
604   * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue.  The
605   * netif_tx_* functions below are used to manipulate this flag.  The
606   * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
607   * queue independently.  The netif_xmit_*stopped functions below are called
608   * to check if the queue has been stopped by the driver or stack (either
609   * of the XOFF bits are set in the state).  Drivers should not need to call
610   * netif_xmit*stopped functions, they should only be using netif_tx_*.
611   */
612  
613  struct netdev_queue {
614  /*
615   * read-mostly part
616   */
617  	struct net_device	*dev;
618  	netdevice_tracker	dev_tracker;
619  
620  	struct Qdisc __rcu	*qdisc;
621  	struct Qdisc __rcu	*qdisc_sleeping;
622  #ifdef CONFIG_SYSFS
623  	struct kobject		kobj;
624  #endif
625  #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
626  	int			numa_node;
627  #endif
628  	unsigned long		tx_maxrate;
629  	/*
630  	 * Number of TX timeouts for this queue
631  	 * (/sys/class/net/DEV/Q/trans_timeout)
632  	 */
633  	atomic_long_t		trans_timeout;
634  
635  	/* Subordinate device that the queue has been assigned to */
636  	struct net_device	*sb_dev;
637  #ifdef CONFIG_XDP_SOCKETS
638  	struct xsk_buff_pool    *pool;
639  #endif
640  /*
641   * write-mostly part
642   */
643  	spinlock_t		_xmit_lock ____cacheline_aligned_in_smp;
644  	int			xmit_lock_owner;
645  	/*
646  	 * Time (in jiffies) of last Tx
647  	 */
648  	unsigned long		trans_start;
649  
650  	unsigned long		state;
651  
652  #ifdef CONFIG_BQL
653  	struct dql		dql;
654  #endif
655  
656  	ANDROID_KABI_RESERVE(1);
657  	ANDROID_KABI_RESERVE(2);
658  	ANDROID_KABI_RESERVE(3);
659  	ANDROID_KABI_RESERVE(4);
660  } ____cacheline_aligned_in_smp;
661  
662  extern int sysctl_fb_tunnels_only_for_init_net;
663  extern int sysctl_devconf_inherit_init_net;
664  
665  /*
666   * sysctl_fb_tunnels_only_for_init_net == 0 : For all netns
667   *                                     == 1 : For initns only
668   *                                     == 2 : For none.
669   */
net_has_fallback_tunnels(const struct net * net)670  static inline bool net_has_fallback_tunnels(const struct net *net)
671  {
672  #if IS_ENABLED(CONFIG_SYSCTL)
673  	int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net);
674  
675  	return !fb_tunnels_only_for_init_net ||
676  		(net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1);
677  #else
678  	return true;
679  #endif
680  }
681  
net_inherit_devconf(void)682  static inline int net_inherit_devconf(void)
683  {
684  #if IS_ENABLED(CONFIG_SYSCTL)
685  	return READ_ONCE(sysctl_devconf_inherit_init_net);
686  #else
687  	return 0;
688  #endif
689  }
690  
netdev_queue_numa_node_read(const struct netdev_queue * q)691  static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
692  {
693  #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
694  	return q->numa_node;
695  #else
696  	return NUMA_NO_NODE;
697  #endif
698  }
699  
netdev_queue_numa_node_write(struct netdev_queue * q,int node)700  static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
701  {
702  #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
703  	q->numa_node = node;
704  #endif
705  }
706  
707  #ifdef CONFIG_RPS
708  /*
709   * This structure holds an RPS map which can be of variable length.  The
710   * map is an array of CPUs.
711   */
712  struct rps_map {
713  	unsigned int len;
714  	struct rcu_head rcu;
715  	u16 cpus[];
716  };
717  #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
718  
719  /*
720   * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
721   * tail pointer for that CPU's input queue at the time of last enqueue, and
722   * a hardware filter index.
723   */
724  struct rps_dev_flow {
725  	u16 cpu;
726  	u16 filter;
727  	unsigned int last_qtail;
728  };
729  #define RPS_NO_FILTER 0xffff
730  
731  /*
732   * The rps_dev_flow_table structure contains a table of flow mappings.
733   */
734  struct rps_dev_flow_table {
735  	unsigned int mask;
736  	struct rcu_head rcu;
737  	struct rps_dev_flow flows[];
738  };
739  #define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
740      ((_num) * sizeof(struct rps_dev_flow)))
741  
742  /*
743   * The rps_sock_flow_table contains mappings of flows to the last CPU
744   * on which they were processed by the application (set in recvmsg).
745   * Each entry is a 32bit value. Upper part is the high-order bits
746   * of flow hash, lower part is CPU number.
747   * rps_cpu_mask is used to partition the space, depending on number of
748   * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
749   * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
750   * meaning we use 32-6=26 bits for the hash.
751   */
752  struct rps_sock_flow_table {
753  	u32	mask;
754  
755  	u32	ents[] ____cacheline_aligned_in_smp;
756  };
757  #define	RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
758  
759  #define RPS_NO_CPU 0xffff
760  
761  extern u32 rps_cpu_mask;
762  extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
763  
rps_record_sock_flow(struct rps_sock_flow_table * table,u32 hash)764  static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
765  					u32 hash)
766  {
767  	if (table && hash) {
768  		unsigned int index = hash & table->mask;
769  		u32 val = hash & ~rps_cpu_mask;
770  
771  		/* We only give a hint, preemption can change CPU under us */
772  		val |= raw_smp_processor_id();
773  
774  		/* The following WRITE_ONCE() is paired with the READ_ONCE()
775  		 * here, and another one in get_rps_cpu().
776  		 */
777  		if (READ_ONCE(table->ents[index]) != val)
778  			WRITE_ONCE(table->ents[index], val);
779  	}
780  }
781  
782  #ifdef CONFIG_RFS_ACCEL
783  bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
784  			 u16 filter_id);
785  #endif
786  #endif /* CONFIG_RPS */
787  
788  /* This structure contains an instance of an RX queue. */
789  struct netdev_rx_queue {
790  	struct xdp_rxq_info		xdp_rxq;
791  #ifdef CONFIG_RPS
792  	struct rps_map __rcu		*rps_map;
793  	struct rps_dev_flow_table __rcu	*rps_flow_table;
794  #endif
795  	struct kobject			kobj;
796  	struct net_device		*dev;
797  	netdevice_tracker		dev_tracker;
798  
799  #ifdef CONFIG_XDP_SOCKETS
800  	struct xsk_buff_pool            *pool;
801  #endif
802  
803  	ANDROID_KABI_RESERVE(1);
804  	ANDROID_KABI_RESERVE(2);
805  	ANDROID_KABI_RESERVE(3);
806  	ANDROID_KABI_RESERVE(4);
807  } ____cacheline_aligned_in_smp;
808  
809  /*
810   * RX queue sysfs structures and functions.
811   */
812  struct rx_queue_attribute {
813  	struct attribute attr;
814  	ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
815  	ssize_t (*store)(struct netdev_rx_queue *queue,
816  			 const char *buf, size_t len);
817  };
818  
819  /* XPS map type and offset of the xps map within net_device->xps_maps[]. */
820  enum xps_map_type {
821  	XPS_CPUS = 0,
822  	XPS_RXQS,
823  	XPS_MAPS_MAX,
824  };
825  
826  #ifdef CONFIG_XPS
827  /*
828   * This structure holds an XPS map which can be of variable length.  The
829   * map is an array of queues.
830   */
831  struct xps_map {
832  	unsigned int len;
833  	unsigned int alloc_len;
834  	struct rcu_head rcu;
835  	u16 queues[];
836  };
837  #define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
838  #define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
839         - sizeof(struct xps_map)) / sizeof(u16))
840  
841  /*
842   * This structure holds all XPS maps for device.  Maps are indexed by CPU.
843   *
844   * We keep track of the number of cpus/rxqs used when the struct is allocated,
845   * in nr_ids. This will help not accessing out-of-bound memory.
846   *
847   * We keep track of the number of traffic classes used when the struct is
848   * allocated, in num_tc. This will be used to navigate the maps, to ensure we're
849   * not crossing its upper bound, as the original dev->num_tc can be updated in
850   * the meantime.
851   */
852  struct xps_dev_maps {
853  	struct rcu_head rcu;
854  	unsigned int nr_ids;
855  	s16 num_tc;
856  	struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
857  };
858  
859  #define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) +	\
860  	(nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
861  
862  #define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
863  	(_rxqs * (_tcs) * sizeof(struct xps_map *)))
864  
865  #endif /* CONFIG_XPS */
866  
867  #define TC_MAX_QUEUE	16
868  #define TC_BITMASK	15
869  /* HW offloaded queuing disciplines txq count and offset maps */
870  struct netdev_tc_txq {
871  	u16 count;
872  	u16 offset;
873  };
874  
875  #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
876  /*
877   * This structure is to hold information about the device
878   * configured to run FCoE protocol stack.
879   */
880  struct netdev_fcoe_hbainfo {
881  	char	manufacturer[64];
882  	char	serial_number[64];
883  	char	hardware_version[64];
884  	char	driver_version[64];
885  	char	optionrom_version[64];
886  	char	firmware_version[64];
887  	char	model[256];
888  	char	model_description[256];
889  };
890  #endif
891  
892  #define MAX_PHYS_ITEM_ID_LEN 32
893  
894  /* This structure holds a unique identifier to identify some
895   * physical item (port for example) used by a netdevice.
896   */
897  struct netdev_phys_item_id {
898  	unsigned char id[MAX_PHYS_ITEM_ID_LEN];
899  	unsigned char id_len;
900  };
901  
netdev_phys_item_id_same(struct netdev_phys_item_id * a,struct netdev_phys_item_id * b)902  static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
903  					    struct netdev_phys_item_id *b)
904  {
905  	return a->id_len == b->id_len &&
906  	       memcmp(a->id, b->id, a->id_len) == 0;
907  }
908  
909  typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
910  				       struct sk_buff *skb,
911  				       struct net_device *sb_dev);
912  
913  enum net_device_path_type {
914  	DEV_PATH_ETHERNET = 0,
915  	DEV_PATH_VLAN,
916  	DEV_PATH_BRIDGE,
917  	DEV_PATH_PPPOE,
918  	DEV_PATH_DSA,
919  	DEV_PATH_MTK_WDMA,
920  };
921  
922  struct net_device_path {
923  	enum net_device_path_type	type;
924  	const struct net_device		*dev;
925  	union {
926  		struct {
927  			u16		id;
928  			__be16		proto;
929  			u8		h_dest[ETH_ALEN];
930  		} encap;
931  		struct {
932  			enum {
933  				DEV_PATH_BR_VLAN_KEEP,
934  				DEV_PATH_BR_VLAN_TAG,
935  				DEV_PATH_BR_VLAN_UNTAG,
936  				DEV_PATH_BR_VLAN_UNTAG_HW,
937  			}		vlan_mode;
938  			u16		vlan_id;
939  			__be16		vlan_proto;
940  		} bridge;
941  		struct {
942  			int port;
943  			u16 proto;
944  		} dsa;
945  		struct {
946  			u8 wdma_idx;
947  			u8 queue;
948  			u16 wcid;
949  			u8 bss;
950  		} mtk_wdma;
951  	};
952  };
953  
954  #define NET_DEVICE_PATH_STACK_MAX	5
955  #define NET_DEVICE_PATH_VLAN_MAX	2
956  
957  struct net_device_path_stack {
958  	int			num_paths;
959  	struct net_device_path	path[NET_DEVICE_PATH_STACK_MAX];
960  };
961  
962  struct net_device_path_ctx {
963  	const struct net_device *dev;
964  	u8			daddr[ETH_ALEN];
965  
966  	int			num_vlans;
967  	struct {
968  		u16		id;
969  		__be16		proto;
970  	} vlan[NET_DEVICE_PATH_VLAN_MAX];
971  };
972  
973  enum tc_setup_type {
974  	TC_QUERY_CAPS,
975  	TC_SETUP_QDISC_MQPRIO,
976  	TC_SETUP_CLSU32,
977  	TC_SETUP_CLSFLOWER,
978  	TC_SETUP_CLSMATCHALL,
979  	TC_SETUP_CLSBPF,
980  	TC_SETUP_BLOCK,
981  	TC_SETUP_QDISC_CBS,
982  	TC_SETUP_QDISC_RED,
983  	TC_SETUP_QDISC_PRIO,
984  	TC_SETUP_QDISC_MQ,
985  	TC_SETUP_QDISC_ETF,
986  	TC_SETUP_ROOT_QDISC,
987  	TC_SETUP_QDISC_GRED,
988  	TC_SETUP_QDISC_TAPRIO,
989  	TC_SETUP_FT,
990  	TC_SETUP_QDISC_ETS,
991  	TC_SETUP_QDISC_TBF,
992  	TC_SETUP_QDISC_FIFO,
993  	TC_SETUP_QDISC_HTB,
994  	TC_SETUP_ACT,
995  };
996  
997  /* These structures hold the attributes of bpf state that are being passed
998   * to the netdevice through the bpf op.
999   */
1000  enum bpf_netdev_command {
1001  	/* Set or clear a bpf program used in the earliest stages of packet
1002  	 * rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
1003  	 * is responsible for calling bpf_prog_put on any old progs that are
1004  	 * stored. In case of error, the callee need not release the new prog
1005  	 * reference, but on success it takes ownership and must bpf_prog_put
1006  	 * when it is no longer used.
1007  	 */
1008  	XDP_SETUP_PROG,
1009  	XDP_SETUP_PROG_HW,
1010  	/* BPF program for offload callbacks, invoked at program load time. */
1011  	BPF_OFFLOAD_MAP_ALLOC,
1012  	BPF_OFFLOAD_MAP_FREE,
1013  	XDP_SETUP_XSK_POOL,
1014  };
1015  
1016  struct bpf_prog_offload_ops;
1017  struct netlink_ext_ack;
1018  struct xdp_umem;
1019  struct xdp_dev_bulk_queue;
1020  struct bpf_xdp_link;
1021  
1022  enum bpf_xdp_mode {
1023  	XDP_MODE_SKB = 0,
1024  	XDP_MODE_DRV = 1,
1025  	XDP_MODE_HW = 2,
1026  	__MAX_XDP_MODE
1027  };
1028  
1029  struct bpf_xdp_entity {
1030  	struct bpf_prog *prog;
1031  	struct bpf_xdp_link *link;
1032  };
1033  
1034  struct netdev_bpf {
1035  	enum bpf_netdev_command command;
1036  	union {
1037  		/* XDP_SETUP_PROG */
1038  		struct {
1039  			u32 flags;
1040  			struct bpf_prog *prog;
1041  			struct netlink_ext_ack *extack;
1042  		};
1043  		/* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
1044  		struct {
1045  			struct bpf_offloaded_map *offmap;
1046  		};
1047  		/* XDP_SETUP_XSK_POOL */
1048  		struct {
1049  			struct xsk_buff_pool *pool;
1050  			u16 queue_id;
1051  		} xsk;
1052  	};
1053  };
1054  
1055  /* Flags for ndo_xsk_wakeup. */
1056  #define XDP_WAKEUP_RX (1 << 0)
1057  #define XDP_WAKEUP_TX (1 << 1)
1058  
1059  #ifdef CONFIG_XFRM_OFFLOAD
1060  struct xfrmdev_ops {
1061  	int	(*xdo_dev_state_add) (struct xfrm_state *x);
1062  	void	(*xdo_dev_state_delete) (struct xfrm_state *x);
1063  	void	(*xdo_dev_state_free) (struct xfrm_state *x);
1064  	bool	(*xdo_dev_offload_ok) (struct sk_buff *skb,
1065  				       struct xfrm_state *x);
1066  	void	(*xdo_dev_state_advance_esn) (struct xfrm_state *x);
1067  
1068  	ANDROID_KABI_RESERVE(1);
1069  	ANDROID_KABI_RESERVE(2);
1070  	ANDROID_KABI_RESERVE(3);
1071  	ANDROID_KABI_RESERVE(4);
1072  };
1073  #endif
1074  
1075  struct dev_ifalias {
1076  	struct rcu_head rcuhead;
1077  	char ifalias[];
1078  };
1079  
1080  struct devlink;
1081  struct tlsdev_ops;
1082  
1083  struct netdev_net_notifier {
1084  	struct list_head list;
1085  	struct notifier_block *nb;
1086  };
1087  
1088  /*
1089   * This structure defines the management hooks for network devices.
1090   * The following hooks can be defined; unless noted otherwise, they are
1091   * optional and can be filled with a null pointer.
1092   *
1093   * int (*ndo_init)(struct net_device *dev);
1094   *     This function is called once when a network device is registered.
1095   *     The network device can use this for any late stage initialization
1096   *     or semantic validation. It can fail with an error code which will
1097   *     be propagated back to register_netdev.
1098   *
1099   * void (*ndo_uninit)(struct net_device *dev);
1100   *     This function is called when device is unregistered or when registration
1101   *     fails. It is not called if init fails.
1102   *
1103   * int (*ndo_open)(struct net_device *dev);
1104   *     This function is called when a network device transitions to the up
1105   *     state.
1106   *
1107   * int (*ndo_stop)(struct net_device *dev);
1108   *     This function is called when a network device transitions to the down
1109   *     state.
1110   *
1111   * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1112   *                               struct net_device *dev);
1113   *	Called when a packet needs to be transmitted.
1114   *	Returns NETDEV_TX_OK.  Can return NETDEV_TX_BUSY, but you should stop
1115   *	the queue before that can happen; it's for obsolete devices and weird
1116   *	corner cases, but the stack really does a non-trivial amount
1117   *	of useless work if you return NETDEV_TX_BUSY.
1118   *	Required; cannot be NULL.
1119   *
1120   * netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1121   *					   struct net_device *dev
1122   *					   netdev_features_t features);
1123   *	Called by core transmit path to determine if device is capable of
1124   *	performing offload operations on a given packet. This is to give
1125   *	the device an opportunity to implement any restrictions that cannot
1126   *	be otherwise expressed by feature flags. The check is called with
1127   *	the set of features that the stack has calculated and it returns
1128   *	those the driver believes to be appropriate.
1129   *
1130   * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
1131   *                         struct net_device *sb_dev);
1132   *	Called to decide which queue to use when device supports multiple
1133   *	transmit queues.
1134   *
1135   * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
1136   *	This function is called to allow device receiver to make
1137   *	changes to configuration when multicast or promiscuous is enabled.
1138   *
1139   * void (*ndo_set_rx_mode)(struct net_device *dev);
1140   *	This function is called device changes address list filtering.
1141   *	If driver handles unicast address filtering, it should set
1142   *	IFF_UNICAST_FLT in its priv_flags.
1143   *
1144   * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
1145   *	This function  is called when the Media Access Control address
1146   *	needs to be changed. If this interface is not defined, the
1147   *	MAC address can not be changed.
1148   *
1149   * int (*ndo_validate_addr)(struct net_device *dev);
1150   *	Test if Media Access Control address is valid for the device.
1151   *
1152   * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1153   *	Old-style ioctl entry point. This is used internally by the
1154   *	appletalk and ieee802154 subsystems but is no longer called by
1155   *	the device ioctl handler.
1156   *
1157   * int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd);
1158   *	Used by the bonding driver for its device specific ioctls:
1159   *	SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE,
1160   *	SIOCBONDSLAVEINFOQUERY, and SIOCBONDINFOQUERY
1161   *
1162   * * int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1163   *	Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG,
1164   *	SIOCSMIIREG, SIOCSHWTSTAMP and SIOCGHWTSTAMP.
1165   *
1166   * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
1167   *	Used to set network devices bus interface parameters. This interface
1168   *	is retained for legacy reasons; new devices should use the bus
1169   *	interface (PCI) for low level management.
1170   *
1171   * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
1172   *	Called when a user wants to change the Maximum Transfer Unit
1173   *	of a device.
1174   *
1175   * void (*ndo_tx_timeout)(struct net_device *dev, unsigned int txqueue);
1176   *	Callback used when the transmitter has not made any progress
1177   *	for dev->watchdog ticks.
1178   *
1179   * void (*ndo_get_stats64)(struct net_device *dev,
1180   *                         struct rtnl_link_stats64 *storage);
1181   * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1182   *	Called when a user wants to get the network device usage
1183   *	statistics. Drivers must do one of the following:
1184   *	1. Define @ndo_get_stats64 to fill in a zero-initialised
1185   *	   rtnl_link_stats64 structure passed by the caller.
1186   *	2. Define @ndo_get_stats to update a net_device_stats structure
1187   *	   (which should normally be dev->stats) and return a pointer to
1188   *	   it. The structure may be changed asynchronously only if each
1189   *	   field is written atomically.
1190   *	3. Update dev->stats asynchronously and atomically, and define
1191   *	   neither operation.
1192   *
1193   * bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
1194   *	Return true if this device supports offload stats of this attr_id.
1195   *
1196   * int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
1197   *	void *attr_data)
1198   *	Get statistics for offload operations by attr_id. Write it into the
1199   *	attr_data pointer.
1200   *
1201   * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
1202   *	If device supports VLAN filtering this function is called when a
1203   *	VLAN id is registered.
1204   *
1205   * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
1206   *	If device supports VLAN filtering this function is called when a
1207   *	VLAN id is unregistered.
1208   *
1209   * void (*ndo_poll_controller)(struct net_device *dev);
1210   *
1211   *	SR-IOV management functions.
1212   * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
1213   * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
1214   *			  u8 qos, __be16 proto);
1215   * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
1216   *			  int max_tx_rate);
1217   * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
1218   * int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
1219   * int (*ndo_get_vf_config)(struct net_device *dev,
1220   *			    int vf, struct ifla_vf_info *ivf);
1221   * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
1222   * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
1223   *			  struct nlattr *port[]);
1224   *
1225   *      Enable or disable the VF ability to query its RSS Redirection Table and
1226   *      Hash Key. This is needed since on some devices VF share this information
1227   *      with PF and querying it may introduce a theoretical security risk.
1228   * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
1229   * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
1230   * int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
1231   *		       void *type_data);
1232   *	Called to setup any 'tc' scheduler, classifier or action on @dev.
1233   *	This is always called from the stack with the rtnl lock held and netif
1234   *	tx queues stopped. This allows the netdevice to perform queue
1235   *	management safely.
1236   *
1237   *	Fiber Channel over Ethernet (FCoE) offload functions.
1238   * int (*ndo_fcoe_enable)(struct net_device *dev);
1239   *	Called when the FCoE protocol stack wants to start using LLD for FCoE
1240   *	so the underlying device can perform whatever needed configuration or
1241   *	initialization to support acceleration of FCoE traffic.
1242   *
1243   * int (*ndo_fcoe_disable)(struct net_device *dev);
1244   *	Called when the FCoE protocol stack wants to stop using LLD for FCoE
1245   *	so the underlying device can perform whatever needed clean-ups to
1246   *	stop supporting acceleration of FCoE traffic.
1247   *
1248   * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
1249   *			     struct scatterlist *sgl, unsigned int sgc);
1250   *	Called when the FCoE Initiator wants to initialize an I/O that
1251   *	is a possible candidate for Direct Data Placement (DDP). The LLD can
1252   *	perform necessary setup and returns 1 to indicate the device is set up
1253   *	successfully to perform DDP on this I/O, otherwise this returns 0.
1254   *
1255   * int (*ndo_fcoe_ddp_done)(struct net_device *dev,  u16 xid);
1256   *	Called when the FCoE Initiator/Target is done with the DDPed I/O as
1257   *	indicated by the FC exchange id 'xid', so the underlying device can
1258   *	clean up and reuse resources for later DDP requests.
1259   *
1260   * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
1261   *			      struct scatterlist *sgl, unsigned int sgc);
1262   *	Called when the FCoE Target wants to initialize an I/O that
1263   *	is a possible candidate for Direct Data Placement (DDP). The LLD can
1264   *	perform necessary setup and returns 1 to indicate the device is set up
1265   *	successfully to perform DDP on this I/O, otherwise this returns 0.
1266   *
1267   * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1268   *			       struct netdev_fcoe_hbainfo *hbainfo);
1269   *	Called when the FCoE Protocol stack wants information on the underlying
1270   *	device. This information is utilized by the FCoE protocol stack to
1271   *	register attributes with Fiber Channel management service as per the
1272   *	FC-GS Fabric Device Management Information(FDMI) specification.
1273   *
1274   * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
1275   *	Called when the underlying device wants to override default World Wide
1276   *	Name (WWN) generation mechanism in FCoE protocol stack to pass its own
1277   *	World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
1278   *	protocol stack to use.
1279   *
1280   *	RFS acceleration.
1281   * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
1282   *			    u16 rxq_index, u32 flow_id);
1283   *	Set hardware filter for RFS.  rxq_index is the target queue index;
1284   *	flow_id is a flow ID to be passed to rps_may_expire_flow() later.
1285   *	Return the filter ID on success, or a negative error code.
1286   *
1287   *	Slave management functions (for bridge, bonding, etc).
1288   * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
1289   *	Called to make another netdev an underling.
1290   *
1291   * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
1292   *	Called to release previously enslaved netdev.
1293   *
1294   * struct net_device *(*ndo_get_xmit_slave)(struct net_device *dev,
1295   *					    struct sk_buff *skb,
1296   *					    bool all_slaves);
1297   *	Get the xmit slave of master device. If all_slaves is true, function
1298   *	assume all the slaves can transmit.
1299   *
1300   *      Feature/offload setting functions.
1301   * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1302   *		netdev_features_t features);
1303   *	Adjusts the requested feature flags according to device-specific
1304   *	constraints, and returns the resulting flags. Must not modify
1305   *	the device state.
1306   *
1307   * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
1308   *	Called to update device configuration to new features. Passed
1309   *	feature set might be less than what was returned by ndo_fix_features()).
1310   *	Must return >0 or -errno if it changed dev->features itself.
1311   *
1312   * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
1313   *		      struct net_device *dev,
1314   *		      const unsigned char *addr, u16 vid, u16 flags,
1315   *		      struct netlink_ext_ack *extack);
1316   *	Adds an FDB entry to dev for addr.
1317   * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
1318   *		      struct net_device *dev,
1319   *		      const unsigned char *addr, u16 vid)
1320   *	Deletes the FDB entry from dev coresponding to addr.
1321   * int (*ndo_fdb_del_bulk)(struct ndmsg *ndm, struct nlattr *tb[],
1322   *			   struct net_device *dev,
1323   *			   u16 vid,
1324   *			   struct netlink_ext_ack *extack);
1325   * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
1326   *		       struct net_device *dev, struct net_device *filter_dev,
1327   *		       int *idx)
1328   *	Used to add FDB entries to dump requests. Implementers should add
1329   *	entries to skb and update idx with the number of entries.
1330   *
1331   * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
1332   *			     u16 flags, struct netlink_ext_ack *extack)
1333   * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
1334   *			     struct net_device *dev, u32 filter_mask,
1335   *			     int nlflags)
1336   * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
1337   *			     u16 flags);
1338   *
1339   * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
1340   *	Called to change device carrier. Soft-devices (like dummy, team, etc)
1341   *	which do not represent real hardware may define this to allow their
1342   *	userspace components to manage their virtual carrier state. Devices
1343   *	that determine carrier state from physical hardware properties (eg
1344   *	network cables) or protocol-dependent mechanisms (eg
1345   *	USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
1346   *
1347   * int (*ndo_get_phys_port_id)(struct net_device *dev,
1348   *			       struct netdev_phys_item_id *ppid);
1349   *	Called to get ID of physical port of this device. If driver does
1350   *	not implement this, it is assumed that the hw is not able to have
1351   *	multiple net devices on single physical port.
1352   *
1353   * int (*ndo_get_port_parent_id)(struct net_device *dev,
1354   *				 struct netdev_phys_item_id *ppid)
1355   *	Called to get the parent ID of the physical port of this device.
1356   *
1357   * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1358   *				 struct net_device *dev)
1359   *	Called by upper layer devices to accelerate switching or other
1360   *	station functionality into hardware. 'pdev is the lowerdev
1361   *	to use for the offload and 'dev' is the net device that will
1362   *	back the offload. Returns a pointer to the private structure
1363   *	the upper layer will maintain.
1364   * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
1365   *	Called by upper layer device to delete the station created
1366   *	by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
1367   *	the station and priv is the structure returned by the add
1368   *	operation.
1369   * int (*ndo_set_tx_maxrate)(struct net_device *dev,
1370   *			     int queue_index, u32 maxrate);
1371   *	Called when a user wants to set a max-rate limitation of specific
1372   *	TX queue.
1373   * int (*ndo_get_iflink)(const struct net_device *dev);
1374   *	Called to get the iflink value of this device.
1375   * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
1376   *	This function is used to get egress tunnel information for given skb.
1377   *	This is useful for retrieving outer tunnel header parameters while
1378   *	sampling packet.
1379   * void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
1380   *	This function is used to specify the headroom that the skb must
1381   *	consider when allocation skb during packet reception. Setting
1382   *	appropriate rx headroom value allows avoiding skb head copy on
1383   *	forward. Setting a negative value resets the rx headroom to the
1384   *	default value.
1385   * int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
1386   *	This function is used to set or query state related to XDP on the
1387   *	netdevice and manage BPF offload. See definition of
1388   *	enum bpf_netdev_command for details.
1389   * int (*ndo_xdp_xmit)(struct net_device *dev, int n, struct xdp_frame **xdp,
1390   *			u32 flags);
1391   *	This function is used to submit @n XDP packets for transmit on a
1392   *	netdevice. Returns number of frames successfully transmitted, frames
1393   *	that got dropped are freed/returned via xdp_return_frame().
1394   *	Returns negative number, means general error invoking ndo, meaning
1395   *	no frames were xmit'ed and core-caller will free all frames.
1396   * struct net_device *(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1397   *					        struct xdp_buff *xdp);
1398   *      Get the xmit slave of master device based on the xdp_buff.
1399   * int (*ndo_xsk_wakeup)(struct net_device *dev, u32 queue_id, u32 flags);
1400   *      This function is used to wake up the softirq, ksoftirqd or kthread
1401   *	responsible for sending and/or receiving packets on a specific
1402   *	queue id bound to an AF_XDP socket. The flags field specifies if
1403   *	only RX, only Tx, or both should be woken up using the flags
1404   *	XDP_WAKEUP_RX and XDP_WAKEUP_TX.
1405   * struct devlink_port *(*ndo_get_devlink_port)(struct net_device *dev);
1406   *	Get devlink port instance associated with a given netdev.
1407   *	Called with a reference on the netdevice and devlink locks only,
1408   *	rtnl_lock is not held.
1409   * int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm *p,
1410   *			 int cmd);
1411   *	Add, change, delete or get information on an IPv4 tunnel.
1412   * struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
1413   *	If a device is paired with a peer device, return the peer instance.
1414   *	The caller must be under RCU read context.
1415   * int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx, struct net_device_path *path);
1416   *     Get the forwarding path to reach the real device from the HW destination address
1417   * ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1418   *			     const struct skb_shared_hwtstamps *hwtstamps,
1419   *			     bool cycles);
1420   *	Get hardware timestamp based on normal/adjustable time or free running
1421   *	cycle counter. This function is required if physical clock supports a
1422   *	free running cycle counter.
1423   */
1424  struct net_device_ops {
1425  	int			(*ndo_init)(struct net_device *dev);
1426  	void			(*ndo_uninit)(struct net_device *dev);
1427  	int			(*ndo_open)(struct net_device *dev);
1428  	int			(*ndo_stop)(struct net_device *dev);
1429  	netdev_tx_t		(*ndo_start_xmit)(struct sk_buff *skb,
1430  						  struct net_device *dev);
1431  	netdev_features_t	(*ndo_features_check)(struct sk_buff *skb,
1432  						      struct net_device *dev,
1433  						      netdev_features_t features);
1434  	u16			(*ndo_select_queue)(struct net_device *dev,
1435  						    struct sk_buff *skb,
1436  						    struct net_device *sb_dev);
1437  	void			(*ndo_change_rx_flags)(struct net_device *dev,
1438  						       int flags);
1439  	void			(*ndo_set_rx_mode)(struct net_device *dev);
1440  	int			(*ndo_set_mac_address)(struct net_device *dev,
1441  						       void *addr);
1442  	int			(*ndo_validate_addr)(struct net_device *dev);
1443  	int			(*ndo_do_ioctl)(struct net_device *dev,
1444  					        struct ifreq *ifr, int cmd);
1445  	int			(*ndo_eth_ioctl)(struct net_device *dev,
1446  						 struct ifreq *ifr, int cmd);
1447  	int			(*ndo_siocbond)(struct net_device *dev,
1448  						struct ifreq *ifr, int cmd);
1449  	int			(*ndo_siocwandev)(struct net_device *dev,
1450  						  struct if_settings *ifs);
1451  	int			(*ndo_siocdevprivate)(struct net_device *dev,
1452  						      struct ifreq *ifr,
1453  						      void __user *data, int cmd);
1454  	int			(*ndo_set_config)(struct net_device *dev,
1455  					          struct ifmap *map);
1456  	int			(*ndo_change_mtu)(struct net_device *dev,
1457  						  int new_mtu);
1458  	int			(*ndo_neigh_setup)(struct net_device *dev,
1459  						   struct neigh_parms *);
1460  	void			(*ndo_tx_timeout) (struct net_device *dev,
1461  						   unsigned int txqueue);
1462  
1463  	void			(*ndo_get_stats64)(struct net_device *dev,
1464  						   struct rtnl_link_stats64 *storage);
1465  	bool			(*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1466  	int			(*ndo_get_offload_stats)(int attr_id,
1467  							 const struct net_device *dev,
1468  							 void *attr_data);
1469  	struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1470  
1471  	int			(*ndo_vlan_rx_add_vid)(struct net_device *dev,
1472  						       __be16 proto, u16 vid);
1473  	int			(*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1474  						        __be16 proto, u16 vid);
1475  #ifdef CONFIG_NET_POLL_CONTROLLER
1476  	void                    (*ndo_poll_controller)(struct net_device *dev);
1477  	int			(*ndo_netpoll_setup)(struct net_device *dev,
1478  						     struct netpoll_info *info);
1479  	void			(*ndo_netpoll_cleanup)(struct net_device *dev);
1480  #endif
1481  	int			(*ndo_set_vf_mac)(struct net_device *dev,
1482  						  int queue, u8 *mac);
1483  	int			(*ndo_set_vf_vlan)(struct net_device *dev,
1484  						   int queue, u16 vlan,
1485  						   u8 qos, __be16 proto);
1486  	int			(*ndo_set_vf_rate)(struct net_device *dev,
1487  						   int vf, int min_tx_rate,
1488  						   int max_tx_rate);
1489  	int			(*ndo_set_vf_spoofchk)(struct net_device *dev,
1490  						       int vf, bool setting);
1491  	int			(*ndo_set_vf_trust)(struct net_device *dev,
1492  						    int vf, bool setting);
1493  	int			(*ndo_get_vf_config)(struct net_device *dev,
1494  						     int vf,
1495  						     struct ifla_vf_info *ivf);
1496  	int			(*ndo_set_vf_link_state)(struct net_device *dev,
1497  							 int vf, int link_state);
1498  	int			(*ndo_get_vf_stats)(struct net_device *dev,
1499  						    int vf,
1500  						    struct ifla_vf_stats
1501  						    *vf_stats);
1502  	int			(*ndo_set_vf_port)(struct net_device *dev,
1503  						   int vf,
1504  						   struct nlattr *port[]);
1505  	int			(*ndo_get_vf_port)(struct net_device *dev,
1506  						   int vf, struct sk_buff *skb);
1507  	int			(*ndo_get_vf_guid)(struct net_device *dev,
1508  						   int vf,
1509  						   struct ifla_vf_guid *node_guid,
1510  						   struct ifla_vf_guid *port_guid);
1511  	int			(*ndo_set_vf_guid)(struct net_device *dev,
1512  						   int vf, u64 guid,
1513  						   int guid_type);
1514  	int			(*ndo_set_vf_rss_query_en)(
1515  						   struct net_device *dev,
1516  						   int vf, bool setting);
1517  	int			(*ndo_setup_tc)(struct net_device *dev,
1518  						enum tc_setup_type type,
1519  						void *type_data);
1520  #if IS_ENABLED(CONFIG_FCOE)
1521  	int			(*ndo_fcoe_enable)(struct net_device *dev);
1522  	int			(*ndo_fcoe_disable)(struct net_device *dev);
1523  	int			(*ndo_fcoe_ddp_setup)(struct net_device *dev,
1524  						      u16 xid,
1525  						      struct scatterlist *sgl,
1526  						      unsigned int sgc);
1527  	int			(*ndo_fcoe_ddp_done)(struct net_device *dev,
1528  						     u16 xid);
1529  	int			(*ndo_fcoe_ddp_target)(struct net_device *dev,
1530  						       u16 xid,
1531  						       struct scatterlist *sgl,
1532  						       unsigned int sgc);
1533  	int			(*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1534  							struct netdev_fcoe_hbainfo *hbainfo);
1535  #endif
1536  
1537  #if IS_ENABLED(CONFIG_LIBFCOE)
1538  #define NETDEV_FCOE_WWNN 0
1539  #define NETDEV_FCOE_WWPN 1
1540  	int			(*ndo_fcoe_get_wwn)(struct net_device *dev,
1541  						    u64 *wwn, int type);
1542  #endif
1543  
1544  #ifdef CONFIG_RFS_ACCEL
1545  	int			(*ndo_rx_flow_steer)(struct net_device *dev,
1546  						     const struct sk_buff *skb,
1547  						     u16 rxq_index,
1548  						     u32 flow_id);
1549  #endif
1550  	int			(*ndo_add_slave)(struct net_device *dev,
1551  						 struct net_device *slave_dev,
1552  						 struct netlink_ext_ack *extack);
1553  	int			(*ndo_del_slave)(struct net_device *dev,
1554  						 struct net_device *slave_dev);
1555  	struct net_device*	(*ndo_get_xmit_slave)(struct net_device *dev,
1556  						      struct sk_buff *skb,
1557  						      bool all_slaves);
1558  	struct net_device*	(*ndo_sk_get_lower_dev)(struct net_device *dev,
1559  							struct sock *sk);
1560  	netdev_features_t	(*ndo_fix_features)(struct net_device *dev,
1561  						    netdev_features_t features);
1562  	int			(*ndo_set_features)(struct net_device *dev,
1563  						    netdev_features_t features);
1564  	int			(*ndo_neigh_construct)(struct net_device *dev,
1565  						       struct neighbour *n);
1566  	void			(*ndo_neigh_destroy)(struct net_device *dev,
1567  						     struct neighbour *n);
1568  
1569  	int			(*ndo_fdb_add)(struct ndmsg *ndm,
1570  					       struct nlattr *tb[],
1571  					       struct net_device *dev,
1572  					       const unsigned char *addr,
1573  					       u16 vid,
1574  					       u16 flags,
1575  					       struct netlink_ext_ack *extack);
1576  	int			(*ndo_fdb_del)(struct ndmsg *ndm,
1577  					       struct nlattr *tb[],
1578  					       struct net_device *dev,
1579  					       const unsigned char *addr,
1580  					       u16 vid, struct netlink_ext_ack *extack);
1581  	int			(*ndo_fdb_del_bulk)(struct ndmsg *ndm,
1582  						    struct nlattr *tb[],
1583  						    struct net_device *dev,
1584  						    u16 vid,
1585  						    struct netlink_ext_ack *extack);
1586  	int			(*ndo_fdb_dump)(struct sk_buff *skb,
1587  						struct netlink_callback *cb,
1588  						struct net_device *dev,
1589  						struct net_device *filter_dev,
1590  						int *idx);
1591  	int			(*ndo_fdb_get)(struct sk_buff *skb,
1592  					       struct nlattr *tb[],
1593  					       struct net_device *dev,
1594  					       const unsigned char *addr,
1595  					       u16 vid, u32 portid, u32 seq,
1596  					       struct netlink_ext_ack *extack);
1597  	int			(*ndo_bridge_setlink)(struct net_device *dev,
1598  						      struct nlmsghdr *nlh,
1599  						      u16 flags,
1600  						      struct netlink_ext_ack *extack);
1601  	int			(*ndo_bridge_getlink)(struct sk_buff *skb,
1602  						      u32 pid, u32 seq,
1603  						      struct net_device *dev,
1604  						      u32 filter_mask,
1605  						      int nlflags);
1606  	int			(*ndo_bridge_dellink)(struct net_device *dev,
1607  						      struct nlmsghdr *nlh,
1608  						      u16 flags);
1609  	int			(*ndo_change_carrier)(struct net_device *dev,
1610  						      bool new_carrier);
1611  	int			(*ndo_get_phys_port_id)(struct net_device *dev,
1612  							struct netdev_phys_item_id *ppid);
1613  	int			(*ndo_get_port_parent_id)(struct net_device *dev,
1614  							  struct netdev_phys_item_id *ppid);
1615  	int			(*ndo_get_phys_port_name)(struct net_device *dev,
1616  							  char *name, size_t len);
1617  	void*			(*ndo_dfwd_add_station)(struct net_device *pdev,
1618  							struct net_device *dev);
1619  	void			(*ndo_dfwd_del_station)(struct net_device *pdev,
1620  							void *priv);
1621  
1622  	int			(*ndo_set_tx_maxrate)(struct net_device *dev,
1623  						      int queue_index,
1624  						      u32 maxrate);
1625  	int			(*ndo_get_iflink)(const struct net_device *dev);
1626  	int			(*ndo_fill_metadata_dst)(struct net_device *dev,
1627  						       struct sk_buff *skb);
1628  	void			(*ndo_set_rx_headroom)(struct net_device *dev,
1629  						       int needed_headroom);
1630  	int			(*ndo_bpf)(struct net_device *dev,
1631  					   struct netdev_bpf *bpf);
1632  	int			(*ndo_xdp_xmit)(struct net_device *dev, int n,
1633  						struct xdp_frame **xdp,
1634  						u32 flags);
1635  	struct net_device *	(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1636  							  struct xdp_buff *xdp);
1637  	int			(*ndo_xsk_wakeup)(struct net_device *dev,
1638  						  u32 queue_id, u32 flags);
1639  	struct devlink_port *	(*ndo_get_devlink_port)(struct net_device *dev);
1640  	int			(*ndo_tunnel_ctl)(struct net_device *dev,
1641  						  struct ip_tunnel_parm *p, int cmd);
1642  	struct net_device *	(*ndo_get_peer_dev)(struct net_device *dev);
1643  	int                     (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
1644                                                           struct net_device_path *path);
1645  	ktime_t			(*ndo_get_tstamp)(struct net_device *dev,
1646  						  const struct skb_shared_hwtstamps *hwtstamps,
1647  						  bool cycles);
1648  
1649  	ANDROID_KABI_RESERVE(1);
1650  	ANDROID_KABI_RESERVE(2);
1651  	ANDROID_KABI_RESERVE(3);
1652  	ANDROID_KABI_RESERVE(4);
1653  	ANDROID_KABI_RESERVE(5);
1654  	ANDROID_KABI_RESERVE(6);
1655  	ANDROID_KABI_RESERVE(7);
1656  	ANDROID_KABI_RESERVE(8);
1657  };
1658  
1659  /**
1660   * enum netdev_priv_flags - &struct net_device priv_flags
1661   *
1662   * These are the &struct net_device, they are only set internally
1663   * by drivers and used in the kernel. These flags are invisible to
1664   * userspace; this means that the order of these flags can change
1665   * during any kernel release.
1666   *
1667   * You should have a pretty good reason to be extending these flags.
1668   *
1669   * @IFF_802_1Q_VLAN: 802.1Q VLAN device
1670   * @IFF_EBRIDGE: Ethernet bridging device
1671   * @IFF_BONDING: bonding master or slave
1672   * @IFF_ISATAP: ISATAP interface (RFC4214)
1673   * @IFF_WAN_HDLC: WAN HDLC device
1674   * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
1675   *	release skb->dst
1676   * @IFF_DONT_BRIDGE: disallow bridging this ether dev
1677   * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
1678   * @IFF_MACVLAN_PORT: device used as macvlan port
1679   * @IFF_BRIDGE_PORT: device used as bridge port
1680   * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
1681   * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
1682   * @IFF_UNICAST_FLT: Supports unicast filtering
1683   * @IFF_TEAM_PORT: device used as team port
1684   * @IFF_SUPP_NOFCS: device supports sending custom FCS
1685   * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
1686   *	change when it's running
1687   * @IFF_MACVLAN: Macvlan device
1688   * @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
1689   *	underlying stacked devices
1690   * @IFF_L3MDEV_MASTER: device is an L3 master device
1691   * @IFF_NO_QUEUE: device can run without qdisc attached
1692   * @IFF_OPENVSWITCH: device is a Open vSwitch master
1693   * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
1694   * @IFF_TEAM: device is a team device
1695   * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
1696   * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
1697   *	entity (i.e. the master device for bridged veth)
1698   * @IFF_MACSEC: device is a MACsec device
1699   * @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
1700   * @IFF_FAILOVER: device is a failover master device
1701   * @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
1702   * @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
1703   * @IFF_LIVE_RENAME_OK: rename is allowed while device is up and running
1704   * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with
1705   *	skb_headlen(skb) == 0 (data starts from frag0)
1706   * @IFF_CHANGE_PROTO_DOWN: device supports setting carrier via IFLA_PROTO_DOWN
1707   */
1708  enum netdev_priv_flags {
1709  	IFF_802_1Q_VLAN			= 1<<0,
1710  	IFF_EBRIDGE			= 1<<1,
1711  	IFF_BONDING			= 1<<2,
1712  	IFF_ISATAP			= 1<<3,
1713  	IFF_WAN_HDLC			= 1<<4,
1714  	IFF_XMIT_DST_RELEASE		= 1<<5,
1715  	IFF_DONT_BRIDGE			= 1<<6,
1716  	IFF_DISABLE_NETPOLL		= 1<<7,
1717  	IFF_MACVLAN_PORT		= 1<<8,
1718  	IFF_BRIDGE_PORT			= 1<<9,
1719  	IFF_OVS_DATAPATH		= 1<<10,
1720  	IFF_TX_SKB_SHARING		= 1<<11,
1721  	IFF_UNICAST_FLT			= 1<<12,
1722  	IFF_TEAM_PORT			= 1<<13,
1723  	IFF_SUPP_NOFCS			= 1<<14,
1724  	IFF_LIVE_ADDR_CHANGE		= 1<<15,
1725  	IFF_MACVLAN			= 1<<16,
1726  	IFF_XMIT_DST_RELEASE_PERM	= 1<<17,
1727  	IFF_L3MDEV_MASTER		= 1<<18,
1728  	IFF_NO_QUEUE			= 1<<19,
1729  	IFF_OPENVSWITCH			= 1<<20,
1730  	IFF_L3MDEV_SLAVE		= 1<<21,
1731  	IFF_TEAM			= 1<<22,
1732  	IFF_RXFH_CONFIGURED		= 1<<23,
1733  	IFF_PHONY_HEADROOM		= 1<<24,
1734  	IFF_MACSEC			= 1<<25,
1735  	IFF_NO_RX_HANDLER		= 1<<26,
1736  	IFF_FAILOVER			= 1<<27,
1737  	IFF_FAILOVER_SLAVE		= 1<<28,
1738  	IFF_L3MDEV_RX_HANDLER		= 1<<29,
1739  	IFF_LIVE_RENAME_OK		= 1<<30,
1740  	IFF_TX_SKB_NO_LINEAR		= BIT_ULL(31),
1741  	IFF_CHANGE_PROTO_DOWN		= BIT_ULL(32),
1742  };
1743  
1744  #define IFF_802_1Q_VLAN			IFF_802_1Q_VLAN
1745  #define IFF_EBRIDGE			IFF_EBRIDGE
1746  #define IFF_BONDING			IFF_BONDING
1747  #define IFF_ISATAP			IFF_ISATAP
1748  #define IFF_WAN_HDLC			IFF_WAN_HDLC
1749  #define IFF_XMIT_DST_RELEASE		IFF_XMIT_DST_RELEASE
1750  #define IFF_DONT_BRIDGE			IFF_DONT_BRIDGE
1751  #define IFF_DISABLE_NETPOLL		IFF_DISABLE_NETPOLL
1752  #define IFF_MACVLAN_PORT		IFF_MACVLAN_PORT
1753  #define IFF_BRIDGE_PORT			IFF_BRIDGE_PORT
1754  #define IFF_OVS_DATAPATH		IFF_OVS_DATAPATH
1755  #define IFF_TX_SKB_SHARING		IFF_TX_SKB_SHARING
1756  #define IFF_UNICAST_FLT			IFF_UNICAST_FLT
1757  #define IFF_TEAM_PORT			IFF_TEAM_PORT
1758  #define IFF_SUPP_NOFCS			IFF_SUPP_NOFCS
1759  #define IFF_LIVE_ADDR_CHANGE		IFF_LIVE_ADDR_CHANGE
1760  #define IFF_MACVLAN			IFF_MACVLAN
1761  #define IFF_XMIT_DST_RELEASE_PERM	IFF_XMIT_DST_RELEASE_PERM
1762  #define IFF_L3MDEV_MASTER		IFF_L3MDEV_MASTER
1763  #define IFF_NO_QUEUE			IFF_NO_QUEUE
1764  #define IFF_OPENVSWITCH			IFF_OPENVSWITCH
1765  #define IFF_L3MDEV_SLAVE		IFF_L3MDEV_SLAVE
1766  #define IFF_TEAM			IFF_TEAM
1767  #define IFF_RXFH_CONFIGURED		IFF_RXFH_CONFIGURED
1768  #define IFF_PHONY_HEADROOM		IFF_PHONY_HEADROOM
1769  #define IFF_MACSEC			IFF_MACSEC
1770  #define IFF_NO_RX_HANDLER		IFF_NO_RX_HANDLER
1771  #define IFF_FAILOVER			IFF_FAILOVER
1772  #define IFF_FAILOVER_SLAVE		IFF_FAILOVER_SLAVE
1773  #define IFF_L3MDEV_RX_HANDLER		IFF_L3MDEV_RX_HANDLER
1774  #define IFF_LIVE_RENAME_OK		IFF_LIVE_RENAME_OK
1775  #define IFF_TX_SKB_NO_LINEAR		IFF_TX_SKB_NO_LINEAR
1776  
1777  /* Specifies the type of the struct net_device::ml_priv pointer */
1778  enum netdev_ml_priv_type {
1779  	ML_PRIV_NONE,
1780  	ML_PRIV_CAN,
1781  };
1782  
1783  /**
1784   *	struct net_device - The DEVICE structure.
1785   *
1786   *	Actually, this whole structure is a big mistake.  It mixes I/O
1787   *	data with strictly "high-level" data, and it has to know about
1788   *	almost every data structure used in the INET module.
1789   *
1790   *	@name:	This is the first field of the "visible" part of this structure
1791   *		(i.e. as seen by users in the "Space.c" file).  It is the name
1792   *		of the interface.
1793   *
1794   *	@name_node:	Name hashlist node
1795   *	@ifalias:	SNMP alias
1796   *	@mem_end:	Shared memory end
1797   *	@mem_start:	Shared memory start
1798   *	@base_addr:	Device I/O address
1799   *	@irq:		Device IRQ number
1800   *
1801   *	@state:		Generic network queuing layer state, see netdev_state_t
1802   *	@dev_list:	The global list of network devices
1803   *	@napi_list:	List entry used for polling NAPI devices
1804   *	@unreg_list:	List entry  when we are unregistering the
1805   *			device; see the function unregister_netdev
1806   *	@close_list:	List entry used when we are closing the device
1807   *	@ptype_all:     Device-specific packet handlers for all protocols
1808   *	@ptype_specific: Device-specific, protocol-specific packet handlers
1809   *
1810   *	@adj_list:	Directly linked devices, like slaves for bonding
1811   *	@features:	Currently active device features
1812   *	@hw_features:	User-changeable features
1813   *
1814   *	@wanted_features:	User-requested features
1815   *	@vlan_features:		Mask of features inheritable by VLAN devices
1816   *
1817   *	@hw_enc_features:	Mask of features inherited by encapsulating devices
1818   *				This field indicates what encapsulation
1819   *				offloads the hardware is capable of doing,
1820   *				and drivers will need to set them appropriately.
1821   *
1822   *	@mpls_features:	Mask of features inheritable by MPLS
1823   *	@gso_partial_features: value(s) from NETIF_F_GSO\*
1824   *
1825   *	@ifindex:	interface index
1826   *	@group:		The group the device belongs to
1827   *
1828   *	@stats:		Statistics struct, which was left as a legacy, use
1829   *			rtnl_link_stats64 instead
1830   *
1831   *	@core_stats:	core networking counters,
1832   *			do not use this in drivers
1833   *	@carrier_up_count:	Number of times the carrier has been up
1834   *	@carrier_down_count:	Number of times the carrier has been down
1835   *
1836   *	@wireless_handlers:	List of functions to handle Wireless Extensions,
1837   *				instead of ioctl,
1838   *				see <net/iw_handler.h> for details.
1839   *	@wireless_data:	Instance data managed by the core of wireless extensions
1840   *
1841   *	@netdev_ops:	Includes several pointers to callbacks,
1842   *			if one wants to override the ndo_*() functions
1843   *	@ethtool_ops:	Management operations
1844   *	@l3mdev_ops:	Layer 3 master device operations
1845   *	@ndisc_ops:	Includes callbacks for different IPv6 neighbour
1846   *			discovery handling. Necessary for e.g. 6LoWPAN.
1847   *	@xfrmdev_ops:	Transformation offload operations
1848   *	@tlsdev_ops:	Transport Layer Security offload operations
1849   *	@header_ops:	Includes callbacks for creating,parsing,caching,etc
1850   *			of Layer 2 headers.
1851   *
1852   *	@flags:		Interface flags (a la BSD)
1853   *	@priv_flags:	Like 'flags' but invisible to userspace,
1854   *			see if.h for the definitions
1855   *	@gflags:	Global flags ( kept as legacy )
1856   *	@padded:	How much padding added by alloc_netdev()
1857   *	@operstate:	RFC2863 operstate
1858   *	@link_mode:	Mapping policy to operstate
1859   *	@if_port:	Selectable AUI, TP, ...
1860   *	@dma:		DMA channel
1861   *	@mtu:		Interface MTU value
1862   *	@min_mtu:	Interface Minimum MTU value
1863   *	@max_mtu:	Interface Maximum MTU value
1864   *	@type:		Interface hardware type
1865   *	@hard_header_len: Maximum hardware header length.
1866   *	@min_header_len:  Minimum hardware header length
1867   *
1868   *	@needed_headroom: Extra headroom the hardware may need, but not in all
1869   *			  cases can this be guaranteed
1870   *	@needed_tailroom: Extra tailroom the hardware may need, but not in all
1871   *			  cases can this be guaranteed. Some cases also use
1872   *			  LL_MAX_HEADER instead to allocate the skb
1873   *
1874   *	interface address info:
1875   *
1876   * 	@perm_addr:		Permanent hw address
1877   * 	@addr_assign_type:	Hw address assignment type
1878   * 	@addr_len:		Hardware address length
1879   *	@upper_level:		Maximum depth level of upper devices.
1880   *	@lower_level:		Maximum depth level of lower devices.
1881   *	@neigh_priv_len:	Used in neigh_alloc()
1882   * 	@dev_id:		Used to differentiate devices that share
1883   * 				the same link layer address
1884   * 	@dev_port:		Used to differentiate devices that share
1885   * 				the same function
1886   *	@addr_list_lock:	XXX: need comments on this one
1887   *	@name_assign_type:	network interface name assignment type
1888   *	@uc_promisc:		Counter that indicates promiscuous mode
1889   *				has been enabled due to the need to listen to
1890   *				additional unicast addresses in a device that
1891   *				does not implement ndo_set_rx_mode()
1892   *	@uc:			unicast mac addresses
1893   *	@mc:			multicast mac addresses
1894   *	@dev_addrs:		list of device hw addresses
1895   *	@queues_kset:		Group of all Kobjects in the Tx and RX queues
1896   *	@promiscuity:		Number of times the NIC is told to work in
1897   *				promiscuous mode; if it becomes 0 the NIC will
1898   *				exit promiscuous mode
1899   *	@allmulti:		Counter, enables or disables allmulticast mode
1900   *
1901   *	@vlan_info:	VLAN info
1902   *	@dsa_ptr:	dsa specific data
1903   *	@tipc_ptr:	TIPC specific data
1904   *	@atalk_ptr:	AppleTalk link
1905   *	@ip_ptr:	IPv4 specific data
1906   *	@ip6_ptr:	IPv6 specific data
1907   *	@ax25_ptr:	AX.25 specific data
1908   *	@ieee80211_ptr:	IEEE 802.11 specific data, assign before registering
1909   *	@ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network
1910   *			 device struct
1911   *	@mpls_ptr:	mpls_dev struct pointer
1912   *	@mctp_ptr:	MCTP specific data
1913   *
1914   *	@dev_addr:	Hw address (before bcast,
1915   *			because most packets are unicast)
1916   *
1917   *	@_rx:			Array of RX queues
1918   *	@num_rx_queues:		Number of RX queues
1919   *				allocated at register_netdev() time
1920   *	@real_num_rx_queues: 	Number of RX queues currently active in device
1921   *	@xdp_prog:		XDP sockets filter program pointer
1922   *	@gro_flush_timeout:	timeout for GRO layer in NAPI
1923   *	@napi_defer_hard_irqs:	If not zero, provides a counter that would
1924   *				allow to avoid NIC hard IRQ, on busy queues.
1925   *
1926   *	@rx_handler:		handler for received packets
1927   *	@rx_handler_data: 	XXX: need comments on this one
1928   *	@miniq_ingress:		ingress/clsact qdisc specific data for
1929   *				ingress processing
1930   *	@ingress_queue:		XXX: need comments on this one
1931   *	@nf_hooks_ingress:	netfilter hooks executed for ingress packets
1932   *	@broadcast:		hw bcast address
1933   *
1934   *	@rx_cpu_rmap:	CPU reverse-mapping for RX completion interrupts,
1935   *			indexed by RX queue number. Assigned by driver.
1936   *			This must only be set if the ndo_rx_flow_steer
1937   *			operation is defined
1938   *	@index_hlist:		Device index hash chain
1939   *
1940   *	@_tx:			Array of TX queues
1941   *	@num_tx_queues:		Number of TX queues allocated at alloc_netdev_mq() time
1942   *	@real_num_tx_queues: 	Number of TX queues currently active in device
1943   *	@qdisc:			Root qdisc from userspace point of view
1944   *	@tx_queue_len:		Max frames per queue allowed
1945   *	@tx_global_lock: 	XXX: need comments on this one
1946   *	@xdp_bulkq:		XDP device bulk queue
1947   *	@xps_maps:		all CPUs/RXQs maps for XPS device
1948   *
1949   *	@xps_maps:	XXX: need comments on this one
1950   *	@miniq_egress:		clsact qdisc specific data for
1951   *				egress processing
1952   *	@nf_hooks_egress:	netfilter hooks executed for egress packets
1953   *	@qdisc_hash:		qdisc hash table
1954   *	@watchdog_timeo:	Represents the timeout that is used by
1955   *				the watchdog (see dev_watchdog())
1956   *	@watchdog_timer:	List of timers
1957   *
1958   *	@proto_down_reason:	reason a netdev interface is held down
1959   *	@pcpu_refcnt:		Number of references to this device
1960   *	@dev_refcnt:		Number of references to this device
1961   *	@refcnt_tracker:	Tracker directory for tracked references to this device
1962   *	@todo_list:		Delayed register/unregister
1963   *	@link_watch_list:	XXX: need comments on this one
1964   *
1965   *	@reg_state:		Register/unregister state machine
1966   *	@dismantle:		Device is going to be freed
1967   *	@rtnl_link_state:	This enum represents the phases of creating
1968   *				a new link
1969   *
1970   *	@needs_free_netdev:	Should unregister perform free_netdev?
1971   *	@priv_destructor:	Called from unregister
1972   *	@npinfo:		XXX: need comments on this one
1973   * 	@nd_net:		Network namespace this network device is inside
1974   *
1975   * 	@ml_priv:	Mid-layer private
1976   *	@ml_priv_type:  Mid-layer private type
1977   * 	@lstats:	Loopback statistics
1978   * 	@tstats:	Tunnel statistics
1979   * 	@dstats:	Dummy statistics
1980   * 	@vstats:	Virtual ethernet statistics
1981   *
1982   *	@garp_port:	GARP
1983   *	@mrp_port:	MRP
1984   *
1985   *	@dm_private:	Drop monitor private
1986   *
1987   *	@dev:		Class/net/name entry
1988   *	@sysfs_groups:	Space for optional device, statistics and wireless
1989   *			sysfs groups
1990   *
1991   *	@sysfs_rx_queue_group:	Space for optional per-rx queue attributes
1992   *	@rtnl_link_ops:	Rtnl_link_ops
1993   *
1994   *	@gso_max_size:	Maximum size of generic segmentation offload
1995   *	@tso_max_size:	Device (as in HW) limit on the max TSO request size
1996   *	@gso_max_segs:	Maximum number of segments that can be passed to the
1997   *			NIC for GSO
1998   *	@tso_max_segs:	Device (as in HW) limit on the max TSO segment count
1999   *
2000   *	@dcbnl_ops:	Data Center Bridging netlink ops
2001   *	@num_tc:	Number of traffic classes in the net device
2002   *	@tc_to_txq:	XXX: need comments on this one
2003   *	@prio_tc_map:	XXX: need comments on this one
2004   *
2005   *	@fcoe_ddp_xid:	Max exchange id for FCoE LRO by ddp
2006   *
2007   *	@priomap:	XXX: need comments on this one
2008   *	@phydev:	Physical device may attach itself
2009   *			for hardware timestamping
2010   *	@sfp_bus:	attached &struct sfp_bus structure.
2011   *
2012   *	@qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
2013   *
2014   *	@proto_down:	protocol port state information can be sent to the
2015   *			switch driver and used to set the phys state of the
2016   *			switch port.
2017   *
2018   *	@wol_enabled:	Wake-on-LAN is enabled
2019   *
2020   *	@threaded:	napi threaded mode is enabled
2021   *
2022   *	@net_notifier_list:	List of per-net netdev notifier block
2023   *				that follow this device when it is moved
2024   *				to another network namespace.
2025   *
2026   *	@macsec_ops:    MACsec offloading ops
2027   *
2028   *	@udp_tunnel_nic_info:	static structure describing the UDP tunnel
2029   *				offload capabilities of the device
2030   *	@udp_tunnel_nic:	UDP tunnel offload state
2031   *	@xdp_state:		stores info on attached XDP BPF programs
2032   *
2033   *	@nested_level:	Used as a parameter of spin_lock_nested() of
2034   *			dev->addr_list_lock.
2035   *	@unlink_list:	As netif_addr_lock() can be called recursively,
2036   *			keep a list of interfaces to be deleted.
2037   *	@gro_max_size:	Maximum size of aggregated packet in generic
2038   *			receive offload (GRO)
2039   *
2040   *	@dev_addr_shadow:	Copy of @dev_addr to catch direct writes.
2041   *	@linkwatch_dev_tracker:	refcount tracker used by linkwatch.
2042   *	@watchdog_dev_tracker:	refcount tracker used by watchdog.
2043   *	@dev_registered_tracker:	tracker for reference held while
2044   *					registered
2045   *	@offload_xstats_l3:	L3 HW stats for this netdevice.
2046   *
2047   *	FIXME: cleanup struct net_device such that network protocol info
2048   *	moves out.
2049   */
2050  
2051  struct net_device {
2052  	char			name[IFNAMSIZ];
2053  	struct netdev_name_node	*name_node;
2054  	struct dev_ifalias	__rcu *ifalias;
2055  	/*
2056  	 *	I/O specific fields
2057  	 *	FIXME: Merge these and struct ifmap into one
2058  	 */
2059  	unsigned long		mem_end;
2060  	unsigned long		mem_start;
2061  	unsigned long		base_addr;
2062  
2063  	/*
2064  	 *	Some hardware also needs these fields (state,dev_list,
2065  	 *	napi_list,unreg_list,close_list) but they are not
2066  	 *	part of the usual set specified in Space.c.
2067  	 */
2068  
2069  	unsigned long		state;
2070  
2071  	struct list_head	dev_list;
2072  	struct list_head	napi_list;
2073  	struct list_head	unreg_list;
2074  	struct list_head	close_list;
2075  	struct list_head	ptype_all;
2076  	struct list_head	ptype_specific;
2077  
2078  	struct {
2079  		struct list_head upper;
2080  		struct list_head lower;
2081  	} adj_list;
2082  
2083  	/* Read-mostly cache-line for fast-path access */
2084  	unsigned int		flags;
2085  	unsigned long long	priv_flags;
2086  	const struct net_device_ops *netdev_ops;
2087  	int			ifindex;
2088  	unsigned short		gflags;
2089  	unsigned short		hard_header_len;
2090  
2091  	/* Note : dev->mtu is often read without holding a lock.
2092  	 * Writers usually hold RTNL.
2093  	 * It is recommended to use READ_ONCE() to annotate the reads,
2094  	 * and to use WRITE_ONCE() to annotate the writes.
2095  	 */
2096  	unsigned int		mtu;
2097  	unsigned short		needed_headroom;
2098  	unsigned short		needed_tailroom;
2099  
2100  	netdev_features_t	features;
2101  	netdev_features_t	hw_features;
2102  	netdev_features_t	wanted_features;
2103  	netdev_features_t	vlan_features;
2104  	netdev_features_t	hw_enc_features;
2105  	netdev_features_t	mpls_features;
2106  	netdev_features_t	gso_partial_features;
2107  
2108  	unsigned int		min_mtu;
2109  	unsigned int		max_mtu;
2110  	unsigned short		type;
2111  	unsigned char		min_header_len;
2112  	unsigned char		name_assign_type;
2113  
2114  	int			group;
2115  
2116  	struct net_device_stats	stats; /* not used by modern drivers */
2117  
2118  	struct net_device_core_stats __percpu *core_stats;
2119  
2120  	/* Stats to monitor link on/off, flapping */
2121  	atomic_t		carrier_up_count;
2122  	atomic_t		carrier_down_count;
2123  
2124  	/* Android KMI hack to allow vendors to have their own wifi changes in modules */
2125  #ifdef __GENKSYMS__
2126  	void			*wireless_handlers;
2127  	void			*wireless_data;
2128  #else
2129  	const struct iw_handler_def *wireless_handlers;
2130  	struct iw_public_data	*wireless_data;
2131  #endif
2132  	const struct ethtool_ops *ethtool_ops;
2133  #ifdef CONFIG_NET_L3_MASTER_DEV
2134  	const struct l3mdev_ops	*l3mdev_ops;
2135  #endif
2136  #if IS_ENABLED(CONFIG_IPV6)
2137  	const struct ndisc_ops *ndisc_ops;
2138  #endif
2139  
2140  #ifdef CONFIG_XFRM_OFFLOAD
2141  	const struct xfrmdev_ops *xfrmdev_ops;
2142  #endif
2143  
2144  #if IS_ENABLED(CONFIG_TLS_DEVICE)
2145  	const struct tlsdev_ops *tlsdev_ops;
2146  #endif
2147  
2148  	const struct header_ops *header_ops;
2149  
2150  	unsigned char		operstate;
2151  	unsigned char		link_mode;
2152  
2153  	unsigned char		if_port;
2154  	unsigned char		dma;
2155  
2156  	/* Interface address info. */
2157  	unsigned char		perm_addr[MAX_ADDR_LEN];
2158  	unsigned char		addr_assign_type;
2159  	unsigned char		addr_len;
2160  	unsigned char		upper_level;
2161  	unsigned char		lower_level;
2162  
2163  	unsigned short		neigh_priv_len;
2164  	unsigned short          dev_id;
2165  	unsigned short          dev_port;
2166  	unsigned short		padded;
2167  
2168  	spinlock_t		addr_list_lock;
2169  	int			irq;
2170  
2171  	struct netdev_hw_addr_list	uc;
2172  	struct netdev_hw_addr_list	mc;
2173  	struct netdev_hw_addr_list	dev_addrs;
2174  
2175  #ifdef CONFIG_SYSFS
2176  	struct kset		*queues_kset;
2177  #endif
2178  #ifdef CONFIG_LOCKDEP
2179  	struct list_head	unlink_list;
2180  #endif
2181  	unsigned int		promiscuity;
2182  	unsigned int		allmulti;
2183  	bool			uc_promisc;
2184  #ifdef CONFIG_LOCKDEP
2185  	unsigned char		nested_level;
2186  #endif
2187  
2188  
2189  	/* Protocol-specific pointers */
2190  
2191  	struct in_device __rcu	*ip_ptr;
2192  	struct inet6_dev __rcu	*ip6_ptr;
2193  #if IS_ENABLED(CONFIG_VLAN_8021Q)
2194  	struct vlan_info __rcu	*vlan_info;
2195  #endif
2196  #if IS_ENABLED(CONFIG_NET_DSA)
2197  	struct dsa_port		*dsa_ptr;
2198  #endif
2199  #if IS_ENABLED(CONFIG_TIPC)
2200  	struct tipc_bearer __rcu *tipc_ptr;
2201  #endif
2202  #if IS_ENABLED(CONFIG_ATALK)
2203  	void 			*atalk_ptr;
2204  #endif
2205  #if IS_ENABLED(CONFIG_AX25)
2206  	void			*ax25_ptr;
2207  #endif
2208  	/* Android KMI hack to allow vendors to have their own wifi changes in modules */
2209  #ifdef __GENKSYMS__
2210  	void			*ieee80211_ptr;
2211  #else
2212  	struct wireless_dev	*ieee80211_ptr;
2213  #endif
2214  
2215  #if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN)
2216  	struct wpan_dev		*ieee802154_ptr;
2217  #endif
2218  #if IS_ENABLED(CONFIG_MPLS_ROUTING)
2219  	struct mpls_dev __rcu	*mpls_ptr;
2220  #endif
2221  #if IS_ENABLED(CONFIG_MCTP)
2222  	struct mctp_dev __rcu	*mctp_ptr;
2223  #endif
2224  
2225  /*
2226   * Cache lines mostly used on receive path (including eth_type_trans())
2227   */
2228  	/* Interface address info used in eth_type_trans() */
2229  	const unsigned char	*dev_addr;
2230  
2231  	struct netdev_rx_queue	*_rx;
2232  	unsigned int		num_rx_queues;
2233  	unsigned int		real_num_rx_queues;
2234  
2235  	struct bpf_prog __rcu	*xdp_prog;
2236  	unsigned long		gro_flush_timeout;
2237  	int			napi_defer_hard_irqs;
2238  #define GRO_LEGACY_MAX_SIZE	65536u
2239  /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2240   * and shinfo->gso_segs is a 16bit field.
2241   */
2242  #define GRO_MAX_SIZE		(8 * 65535u)
2243  	unsigned int		gro_max_size;
2244  	rx_handler_func_t __rcu	*rx_handler;
2245  	void __rcu		*rx_handler_data;
2246  
2247  #ifdef CONFIG_NET_CLS_ACT
2248  	struct mini_Qdisc __rcu	*miniq_ingress;
2249  #endif
2250  	struct netdev_queue __rcu *ingress_queue;
2251  #ifdef CONFIG_NETFILTER_INGRESS
2252  	struct nf_hook_entries __rcu *nf_hooks_ingress;
2253  #endif
2254  
2255  	unsigned char		broadcast[MAX_ADDR_LEN];
2256  #ifdef CONFIG_RFS_ACCEL
2257  	struct cpu_rmap		*rx_cpu_rmap;
2258  #endif
2259  	struct hlist_node	index_hlist;
2260  
2261  /*
2262   * Cache lines mostly used on transmit path
2263   */
2264  	struct netdev_queue	*_tx ____cacheline_aligned_in_smp;
2265  	unsigned int		num_tx_queues;
2266  	unsigned int		real_num_tx_queues;
2267  	struct Qdisc __rcu	*qdisc;
2268  	unsigned int		tx_queue_len;
2269  	spinlock_t		tx_global_lock;
2270  
2271  	struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
2272  
2273  #ifdef CONFIG_XPS
2274  	struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX];
2275  #endif
2276  #ifdef CONFIG_NET_CLS_ACT
2277  	struct mini_Qdisc __rcu	*miniq_egress;
2278  #endif
2279  #ifdef CONFIG_NETFILTER_EGRESS
2280  	struct nf_hook_entries __rcu *nf_hooks_egress;
2281  #endif
2282  
2283  #ifdef CONFIG_NET_SCHED
2284  	DECLARE_HASHTABLE	(qdisc_hash, 4);
2285  #endif
2286  	/* These may be needed for future network-power-down code. */
2287  	struct timer_list	watchdog_timer;
2288  	int			watchdog_timeo;
2289  
2290  	u32                     proto_down_reason;
2291  
2292  	struct list_head	todo_list;
2293  
2294  #ifdef CONFIG_PCPU_DEV_REFCNT
2295  	int __percpu		*pcpu_refcnt;
2296  #else
2297  	refcount_t		dev_refcnt;
2298  #endif
2299  	struct ref_tracker_dir	refcnt_tracker;
2300  
2301  	struct list_head	link_watch_list;
2302  
2303  	enum { NETREG_UNINITIALIZED=0,
2304  	       NETREG_REGISTERED,	/* completed register_netdevice */
2305  	       NETREG_UNREGISTERING,	/* called unregister_netdevice */
2306  	       NETREG_UNREGISTERED,	/* completed unregister todo */
2307  	       NETREG_RELEASED,		/* called free_netdev */
2308  	       NETREG_DUMMY,		/* dummy device for NAPI poll */
2309  	} reg_state:8;
2310  
2311  	bool dismantle;
2312  
2313  	enum {
2314  		RTNL_LINK_INITIALIZED,
2315  		RTNL_LINK_INITIALIZING,
2316  	} rtnl_link_state:16;
2317  
2318  	bool needs_free_netdev;
2319  	void (*priv_destructor)(struct net_device *dev);
2320  
2321  #ifdef CONFIG_NETPOLL
2322  	struct netpoll_info __rcu	*npinfo;
2323  #endif
2324  
2325  	possible_net_t			nd_net;
2326  
2327  	/* mid-layer private */
2328  	void				*ml_priv;
2329  	enum netdev_ml_priv_type	ml_priv_type;
2330  
2331  	union {
2332  		struct pcpu_lstats __percpu		*lstats;
2333  		struct pcpu_sw_netstats __percpu	*tstats;
2334  		struct pcpu_dstats __percpu		*dstats;
2335  	};
2336  
2337  #if IS_ENABLED(CONFIG_GARP)
2338  	struct garp_port __rcu	*garp_port;
2339  #endif
2340  #if IS_ENABLED(CONFIG_MRP)
2341  	struct mrp_port __rcu	*mrp_port;
2342  #endif
2343  #if IS_ENABLED(CONFIG_NET_DROP_MONITOR)
2344  	struct dm_hw_stat_delta __rcu *dm_private;
2345  #endif
2346  	struct device		dev;
2347  	const struct attribute_group *sysfs_groups[4];
2348  	const struct attribute_group *sysfs_rx_queue_group;
2349  
2350  	const struct rtnl_link_ops *rtnl_link_ops;
2351  
2352  	/* for setting kernel sock attribute on TCP connection setup */
2353  #define GSO_MAX_SEGS		65535u
2354  #define GSO_LEGACY_MAX_SIZE	65536u
2355  /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2356   * and shinfo->gso_segs is a 16bit field.
2357   */
2358  #define GSO_MAX_SIZE		(8 * GSO_MAX_SEGS)
2359  
2360  	unsigned int		gso_max_size;
2361  #define TSO_LEGACY_MAX_SIZE	65536
2362  #define TSO_MAX_SIZE		UINT_MAX
2363  	unsigned int		tso_max_size;
2364  	u16			gso_max_segs;
2365  #define TSO_MAX_SEGS		U16_MAX
2366  	u16			tso_max_segs;
2367  
2368  #ifdef CONFIG_DCB
2369  	const struct dcbnl_rtnl_ops *dcbnl_ops;
2370  #endif
2371  	s16			num_tc;
2372  	struct netdev_tc_txq	tc_to_txq[TC_MAX_QUEUE];
2373  	u8			prio_tc_map[TC_BITMASK + 1];
2374  
2375  #if IS_ENABLED(CONFIG_FCOE)
2376  	unsigned int		fcoe_ddp_xid;
2377  #endif
2378  #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2379  	struct netprio_map __rcu *priomap;
2380  #endif
2381  	struct phy_device	*phydev;
2382  	struct sfp_bus		*sfp_bus;
2383  	struct lock_class_key	*qdisc_tx_busylock;
2384  	bool			proto_down;
2385  	unsigned		wol_enabled:1;
2386  	unsigned		threaded:1;
2387  
2388  	struct list_head	net_notifier_list;
2389  
2390  #if IS_ENABLED(CONFIG_MACSEC)
2391  	/* MACsec management functions */
2392  	const struct macsec_ops *macsec_ops;
2393  #endif
2394  	const struct udp_tunnel_nic_info	*udp_tunnel_nic_info;
2395  	struct udp_tunnel_nic	*udp_tunnel_nic;
2396  
2397  	/* protected by rtnl_lock */
2398  	struct bpf_xdp_entity	xdp_state[__MAX_XDP_MODE];
2399  
2400  	u8 dev_addr_shadow[MAX_ADDR_LEN];
2401  	netdevice_tracker	linkwatch_dev_tracker;
2402  	netdevice_tracker	watchdog_dev_tracker;
2403  	netdevice_tracker	dev_registered_tracker;
2404  	struct rtnl_hw_stats64	*offload_xstats_l3;
2405  
2406  	ANDROID_KABI_RESERVE(1);
2407  	ANDROID_KABI_RESERVE(2);
2408  	ANDROID_KABI_RESERVE(3);
2409  	ANDROID_KABI_RESERVE(4);
2410  	ANDROID_KABI_RESERVE(5);
2411  	ANDROID_KABI_RESERVE(6);
2412  	ANDROID_KABI_RESERVE(7);
2413  	ANDROID_KABI_RESERVE(8);
2414  };
2415  #define to_net_dev(d) container_of(d, struct net_device, dev)
2416  
netif_elide_gro(const struct net_device * dev)2417  static inline bool netif_elide_gro(const struct net_device *dev)
2418  {
2419  	if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
2420  		return true;
2421  	return false;
2422  }
2423  
2424  #define	NETDEV_ALIGN		32
2425  
2426  static inline
netdev_get_prio_tc_map(const struct net_device * dev,u32 prio)2427  int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
2428  {
2429  	return dev->prio_tc_map[prio & TC_BITMASK];
2430  }
2431  
2432  static inline
netdev_set_prio_tc_map(struct net_device * dev,u8 prio,u8 tc)2433  int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
2434  {
2435  	if (tc >= dev->num_tc)
2436  		return -EINVAL;
2437  
2438  	dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
2439  	return 0;
2440  }
2441  
2442  int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
2443  void netdev_reset_tc(struct net_device *dev);
2444  int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
2445  int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
2446  
2447  static inline
netdev_get_num_tc(struct net_device * dev)2448  int netdev_get_num_tc(struct net_device *dev)
2449  {
2450  	return dev->num_tc;
2451  }
2452  
net_prefetch(void * p)2453  static inline void net_prefetch(void *p)
2454  {
2455  	prefetch(p);
2456  #if L1_CACHE_BYTES < 128
2457  	prefetch((u8 *)p + L1_CACHE_BYTES);
2458  #endif
2459  }
2460  
net_prefetchw(void * p)2461  static inline void net_prefetchw(void *p)
2462  {
2463  	prefetchw(p);
2464  #if L1_CACHE_BYTES < 128
2465  	prefetchw((u8 *)p + L1_CACHE_BYTES);
2466  #endif
2467  }
2468  
2469  void netdev_unbind_sb_channel(struct net_device *dev,
2470  			      struct net_device *sb_dev);
2471  int netdev_bind_sb_channel_queue(struct net_device *dev,
2472  				 struct net_device *sb_dev,
2473  				 u8 tc, u16 count, u16 offset);
2474  int netdev_set_sb_channel(struct net_device *dev, u16 channel);
netdev_get_sb_channel(struct net_device * dev)2475  static inline int netdev_get_sb_channel(struct net_device *dev)
2476  {
2477  	return max_t(int, -dev->num_tc, 0);
2478  }
2479  
2480  static inline
netdev_get_tx_queue(const struct net_device * dev,unsigned int index)2481  struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
2482  					 unsigned int index)
2483  {
2484  	DEBUG_NET_WARN_ON_ONCE(index >= dev->num_tx_queues);
2485  	return &dev->_tx[index];
2486  }
2487  
skb_get_tx_queue(const struct net_device * dev,const struct sk_buff * skb)2488  static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2489  						    const struct sk_buff *skb)
2490  {
2491  	return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2492  }
2493  
netdev_for_each_tx_queue(struct net_device * dev,void (* f)(struct net_device *,struct netdev_queue *,void *),void * arg)2494  static inline void netdev_for_each_tx_queue(struct net_device *dev,
2495  					    void (*f)(struct net_device *,
2496  						      struct netdev_queue *,
2497  						      void *),
2498  					    void *arg)
2499  {
2500  	unsigned int i;
2501  
2502  	for (i = 0; i < dev->num_tx_queues; i++)
2503  		f(dev, &dev->_tx[i], arg);
2504  }
2505  
2506  #define netdev_lockdep_set_classes(dev)				\
2507  {								\
2508  	static struct lock_class_key qdisc_tx_busylock_key;	\
2509  	static struct lock_class_key qdisc_xmit_lock_key;	\
2510  	static struct lock_class_key dev_addr_list_lock_key;	\
2511  	unsigned int i;						\
2512  								\
2513  	(dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key;	\
2514  	lockdep_set_class(&(dev)->addr_list_lock,		\
2515  			  &dev_addr_list_lock_key);		\
2516  	for (i = 0; i < (dev)->num_tx_queues; i++)		\
2517  		lockdep_set_class(&(dev)->_tx[i]._xmit_lock,	\
2518  				  &qdisc_xmit_lock_key);	\
2519  }
2520  
2521  u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
2522  		     struct net_device *sb_dev);
2523  struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
2524  					 struct sk_buff *skb,
2525  					 struct net_device *sb_dev);
2526  
2527  /* returns the headroom that the master device needs to take in account
2528   * when forwarding to this dev
2529   */
netdev_get_fwd_headroom(struct net_device * dev)2530  static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2531  {
2532  	return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2533  }
2534  
netdev_set_rx_headroom(struct net_device * dev,int new_hr)2535  static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2536  {
2537  	if (dev->netdev_ops->ndo_set_rx_headroom)
2538  		dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2539  }
2540  
2541  /* set the device rx headroom to the dev's default */
netdev_reset_rx_headroom(struct net_device * dev)2542  static inline void netdev_reset_rx_headroom(struct net_device *dev)
2543  {
2544  	netdev_set_rx_headroom(dev, -1);
2545  }
2546  
netdev_get_ml_priv(struct net_device * dev,enum netdev_ml_priv_type type)2547  static inline void *netdev_get_ml_priv(struct net_device *dev,
2548  				       enum netdev_ml_priv_type type)
2549  {
2550  	if (dev->ml_priv_type != type)
2551  		return NULL;
2552  
2553  	return dev->ml_priv;
2554  }
2555  
netdev_set_ml_priv(struct net_device * dev,void * ml_priv,enum netdev_ml_priv_type type)2556  static inline void netdev_set_ml_priv(struct net_device *dev,
2557  				      void *ml_priv,
2558  				      enum netdev_ml_priv_type type)
2559  {
2560  	WARN(dev->ml_priv_type && dev->ml_priv_type != type,
2561  	     "Overwriting already set ml_priv_type (%u) with different ml_priv_type (%u)!\n",
2562  	     dev->ml_priv_type, type);
2563  	WARN(!dev->ml_priv_type && dev->ml_priv,
2564  	     "Overwriting already set ml_priv and ml_priv_type is ML_PRIV_NONE!\n");
2565  
2566  	dev->ml_priv = ml_priv;
2567  	dev->ml_priv_type = type;
2568  }
2569  
2570  /*
2571   * Net namespace inlines
2572   */
2573  static inline
dev_net(const struct net_device * dev)2574  struct net *dev_net(const struct net_device *dev)
2575  {
2576  	return read_pnet(&dev->nd_net);
2577  }
2578  
2579  static inline
dev_net_set(struct net_device * dev,struct net * net)2580  void dev_net_set(struct net_device *dev, struct net *net)
2581  {
2582  	write_pnet(&dev->nd_net, net);
2583  }
2584  
2585  /**
2586   *	netdev_priv - access network device private data
2587   *	@dev: network device
2588   *
2589   * Get network device private data
2590   */
netdev_priv(const struct net_device * dev)2591  static inline void *netdev_priv(const struct net_device *dev)
2592  {
2593  	return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
2594  }
2595  
2596  /* Set the sysfs physical device reference for the network logical device
2597   * if set prior to registration will cause a symlink during initialization.
2598   */
2599  #define SET_NETDEV_DEV(net, pdev)	((net)->dev.parent = (pdev))
2600  
2601  /* Set the sysfs device type for the network logical device to allow
2602   * fine-grained identification of different network device types. For
2603   * example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
2604   */
2605  #define SET_NETDEV_DEVTYPE(net, devtype)	((net)->dev.type = (devtype))
2606  
2607  /* Default NAPI poll() weight
2608   * Device drivers are strongly advised to not use bigger value
2609   */
2610  #define NAPI_POLL_WEIGHT 64
2611  
2612  void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
2613  			   int (*poll)(struct napi_struct *, int), int weight);
2614  
2615  /**
2616   * netif_napi_add() - initialize a NAPI context
2617   * @dev:  network device
2618   * @napi: NAPI context
2619   * @poll: polling function
2620   *
2621   * netif_napi_add() must be used to initialize a NAPI context prior to calling
2622   * *any* of the other NAPI-related functions.
2623   */
2624  static inline void
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2625  netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2626  	       int (*poll)(struct napi_struct *, int))
2627  {
2628  	netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2629  }
2630  
2631  static inline void
netif_napi_add_tx_weight(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)2632  netif_napi_add_tx_weight(struct net_device *dev,
2633  			 struct napi_struct *napi,
2634  			 int (*poll)(struct napi_struct *, int),
2635  			 int weight)
2636  {
2637  	set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2638  	netif_napi_add_weight(dev, napi, poll, weight);
2639  }
2640  
2641  /**
2642   * netif_napi_add_tx() - initialize a NAPI context to be used for Tx only
2643   * @dev:  network device
2644   * @napi: NAPI context
2645   * @poll: polling function
2646   *
2647   * This variant of netif_napi_add() should be used from drivers using NAPI
2648   * to exclusively poll a TX queue.
2649   * This will avoid we add it into napi_hash[], thus polluting this hash table.
2650   */
netif_napi_add_tx(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2651  static inline void netif_napi_add_tx(struct net_device *dev,
2652  				     struct napi_struct *napi,
2653  				     int (*poll)(struct napi_struct *, int))
2654  {
2655  	netif_napi_add_tx_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2656  }
2657  
2658  /**
2659   *  __netif_napi_del - remove a NAPI context
2660   *  @napi: NAPI context
2661   *
2662   * Warning: caller must observe RCU grace period before freeing memory
2663   * containing @napi. Drivers might want to call this helper to combine
2664   * all the needed RCU grace periods into a single one.
2665   */
2666  void __netif_napi_del(struct napi_struct *napi);
2667  
2668  /**
2669   *  netif_napi_del - remove a NAPI context
2670   *  @napi: NAPI context
2671   *
2672   *  netif_napi_del() removes a NAPI context from the network device NAPI list
2673   */
netif_napi_del(struct napi_struct * napi)2674  static inline void netif_napi_del(struct napi_struct *napi)
2675  {
2676  	__netif_napi_del(napi);
2677  	synchronize_net();
2678  }
2679  
2680  struct packet_type {
2681  	__be16			type;	/* This is really htons(ether_type). */
2682  	bool			ignore_outgoing;
2683  	struct net_device	*dev;	/* NULL is wildcarded here	     */
2684  	netdevice_tracker	dev_tracker;
2685  	int			(*func) (struct sk_buff *,
2686  					 struct net_device *,
2687  					 struct packet_type *,
2688  					 struct net_device *);
2689  	void			(*list_func) (struct list_head *,
2690  					      struct packet_type *,
2691  					      struct net_device *);
2692  	bool			(*id_match)(struct packet_type *ptype,
2693  					    struct sock *sk);
2694  	struct net		*af_packet_net;
2695  	void			*af_packet_priv;
2696  	struct list_head	list;
2697  
2698  	ANDROID_KABI_RESERVE(1);
2699  	ANDROID_KABI_RESERVE(2);
2700  	ANDROID_KABI_RESERVE(3);
2701  	ANDROID_KABI_RESERVE(4);
2702  };
2703  
2704  struct offload_callbacks {
2705  	struct sk_buff		*(*gso_segment)(struct sk_buff *skb,
2706  						netdev_features_t features);
2707  	struct sk_buff		*(*gro_receive)(struct list_head *head,
2708  						struct sk_buff *skb);
2709  	int			(*gro_complete)(struct sk_buff *skb, int nhoff);
2710  };
2711  
2712  struct packet_offload {
2713  	__be16			 type;	/* This is really htons(ether_type). */
2714  	u16			 priority;
2715  	struct offload_callbacks callbacks;
2716  	struct list_head	 list;
2717  };
2718  
2719  /* often modified stats are per-CPU, other are shared (netdev->stats) */
2720  struct pcpu_sw_netstats {
2721  	u64_stats_t		rx_packets;
2722  	u64_stats_t		rx_bytes;
2723  	u64_stats_t		tx_packets;
2724  	u64_stats_t		tx_bytes;
2725  	struct u64_stats_sync   syncp;
2726  } __aligned(4 * sizeof(u64));
2727  
2728  struct pcpu_lstats {
2729  	u64_stats_t packets;
2730  	u64_stats_t bytes;
2731  	struct u64_stats_sync syncp;
2732  } __aligned(2 * sizeof(u64));
2733  
2734  void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);
2735  
dev_sw_netstats_rx_add(struct net_device * dev,unsigned int len)2736  static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
2737  {
2738  	struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2739  
2740  	u64_stats_update_begin(&tstats->syncp);
2741  	u64_stats_add(&tstats->rx_bytes, len);
2742  	u64_stats_inc(&tstats->rx_packets);
2743  	u64_stats_update_end(&tstats->syncp);
2744  }
2745  
dev_sw_netstats_tx_add(struct net_device * dev,unsigned int packets,unsigned int len)2746  static inline void dev_sw_netstats_tx_add(struct net_device *dev,
2747  					  unsigned int packets,
2748  					  unsigned int len)
2749  {
2750  	struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2751  
2752  	u64_stats_update_begin(&tstats->syncp);
2753  	u64_stats_add(&tstats->tx_bytes, len);
2754  	u64_stats_add(&tstats->tx_packets, packets);
2755  	u64_stats_update_end(&tstats->syncp);
2756  }
2757  
dev_lstats_add(struct net_device * dev,unsigned int len)2758  static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
2759  {
2760  	struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);
2761  
2762  	u64_stats_update_begin(&lstats->syncp);
2763  	u64_stats_add(&lstats->bytes, len);
2764  	u64_stats_inc(&lstats->packets);
2765  	u64_stats_update_end(&lstats->syncp);
2766  }
2767  
2768  #define __netdev_alloc_pcpu_stats(type, gfp)				\
2769  ({									\
2770  	typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2771  	if (pcpu_stats)	{						\
2772  		int __cpu;						\
2773  		for_each_possible_cpu(__cpu) {				\
2774  			typeof(type) *stat;				\
2775  			stat = per_cpu_ptr(pcpu_stats, __cpu);		\
2776  			u64_stats_init(&stat->syncp);			\
2777  		}							\
2778  	}								\
2779  	pcpu_stats;							\
2780  })
2781  
2782  #define netdev_alloc_pcpu_stats(type)					\
2783  	__netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2784  
2785  #define devm_netdev_alloc_pcpu_stats(dev, type)				\
2786  ({									\
2787  	typeof(type) __percpu *pcpu_stats = devm_alloc_percpu(dev, type);\
2788  	if (pcpu_stats) {						\
2789  		int __cpu;						\
2790  		for_each_possible_cpu(__cpu) {				\
2791  			typeof(type) *stat;				\
2792  			stat = per_cpu_ptr(pcpu_stats, __cpu);		\
2793  			u64_stats_init(&stat->syncp);			\
2794  		}							\
2795  	}								\
2796  	pcpu_stats;							\
2797  })
2798  
2799  enum netdev_lag_tx_type {
2800  	NETDEV_LAG_TX_TYPE_UNKNOWN,
2801  	NETDEV_LAG_TX_TYPE_RANDOM,
2802  	NETDEV_LAG_TX_TYPE_BROADCAST,
2803  	NETDEV_LAG_TX_TYPE_ROUNDROBIN,
2804  	NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
2805  	NETDEV_LAG_TX_TYPE_HASH,
2806  };
2807  
2808  enum netdev_lag_hash {
2809  	NETDEV_LAG_HASH_NONE,
2810  	NETDEV_LAG_HASH_L2,
2811  	NETDEV_LAG_HASH_L34,
2812  	NETDEV_LAG_HASH_L23,
2813  	NETDEV_LAG_HASH_E23,
2814  	NETDEV_LAG_HASH_E34,
2815  	NETDEV_LAG_HASH_VLAN_SRCMAC,
2816  	NETDEV_LAG_HASH_UNKNOWN,
2817  };
2818  
2819  struct netdev_lag_upper_info {
2820  	enum netdev_lag_tx_type tx_type;
2821  	enum netdev_lag_hash hash_type;
2822  };
2823  
2824  struct netdev_lag_lower_state_info {
2825  	u8 link_up : 1,
2826  	   tx_enabled : 1;
2827  };
2828  
2829  #include <linux/notifier.h>
2830  
2831  /* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
2832   * and the rtnetlink notification exclusion list in rtnetlink_event() when
2833   * adding new types.
2834   */
2835  enum netdev_cmd {
2836  	NETDEV_UP	= 1,	/* For now you can't veto a device up/down */
2837  	NETDEV_DOWN,
2838  	NETDEV_REBOOT,		/* Tell a protocol stack a network interface
2839  				   detected a hardware crash and restarted
2840  				   - we can use this eg to kick tcp sessions
2841  				   once done */
2842  	NETDEV_CHANGE,		/* Notify device state change */
2843  	NETDEV_REGISTER,
2844  	NETDEV_UNREGISTER,
2845  	NETDEV_CHANGEMTU,	/* notify after mtu change happened */
2846  	NETDEV_CHANGEADDR,	/* notify after the address change */
2847  	NETDEV_PRE_CHANGEADDR,	/* notify before the address change */
2848  	NETDEV_GOING_DOWN,
2849  	NETDEV_CHANGENAME,
2850  	NETDEV_FEAT_CHANGE,
2851  	NETDEV_BONDING_FAILOVER,
2852  	NETDEV_PRE_UP,
2853  	NETDEV_PRE_TYPE_CHANGE,
2854  	NETDEV_POST_TYPE_CHANGE,
2855  	NETDEV_POST_INIT,
2856  	NETDEV_RELEASE,
2857  	NETDEV_NOTIFY_PEERS,
2858  	NETDEV_JOIN,
2859  	NETDEV_CHANGEUPPER,
2860  	NETDEV_RESEND_IGMP,
2861  	NETDEV_PRECHANGEMTU,	/* notify before mtu change happened */
2862  	NETDEV_CHANGEINFODATA,
2863  	NETDEV_BONDING_INFO,
2864  	NETDEV_PRECHANGEUPPER,
2865  	NETDEV_CHANGELOWERSTATE,
2866  	NETDEV_UDP_TUNNEL_PUSH_INFO,
2867  	NETDEV_UDP_TUNNEL_DROP_INFO,
2868  	NETDEV_CHANGE_TX_QUEUE_LEN,
2869  	NETDEV_CVLAN_FILTER_PUSH_INFO,
2870  	NETDEV_CVLAN_FILTER_DROP_INFO,
2871  	NETDEV_SVLAN_FILTER_PUSH_INFO,
2872  	NETDEV_SVLAN_FILTER_DROP_INFO,
2873  	NETDEV_OFFLOAD_XSTATS_ENABLE,
2874  	NETDEV_OFFLOAD_XSTATS_DISABLE,
2875  	NETDEV_OFFLOAD_XSTATS_REPORT_USED,
2876  	NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
2877  };
2878  const char *netdev_cmd_to_name(enum netdev_cmd cmd);
2879  
2880  int register_netdevice_notifier(struct notifier_block *nb);
2881  int unregister_netdevice_notifier(struct notifier_block *nb);
2882  int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
2883  int unregister_netdevice_notifier_net(struct net *net,
2884  				      struct notifier_block *nb);
2885  int register_netdevice_notifier_dev_net(struct net_device *dev,
2886  					struct notifier_block *nb,
2887  					struct netdev_net_notifier *nn);
2888  int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2889  					  struct notifier_block *nb,
2890  					  struct netdev_net_notifier *nn);
2891  
2892  struct netdev_notifier_info {
2893  	struct net_device	*dev;
2894  	struct netlink_ext_ack	*extack;
2895  };
2896  
2897  struct netdev_notifier_info_ext {
2898  	struct netdev_notifier_info info; /* must be first */
2899  	union {
2900  		u32 mtu;
2901  	} ext;
2902  };
2903  
2904  struct netdev_notifier_change_info {
2905  	struct netdev_notifier_info info; /* must be first */
2906  	unsigned int flags_changed;
2907  };
2908  
2909  struct netdev_notifier_changeupper_info {
2910  	struct netdev_notifier_info info; /* must be first */
2911  	struct net_device *upper_dev; /* new upper dev */
2912  	bool master; /* is upper dev master */
2913  	bool linking; /* is the notification for link or unlink */
2914  	void *upper_info; /* upper dev info */
2915  };
2916  
2917  struct netdev_notifier_changelowerstate_info {
2918  	struct netdev_notifier_info info; /* must be first */
2919  	void *lower_state_info; /* is lower dev state */
2920  };
2921  
2922  struct netdev_notifier_pre_changeaddr_info {
2923  	struct netdev_notifier_info info; /* must be first */
2924  	const unsigned char *dev_addr;
2925  };
2926  
2927  enum netdev_offload_xstats_type {
2928  	NETDEV_OFFLOAD_XSTATS_TYPE_L3 = 1,
2929  };
2930  
2931  struct netdev_notifier_offload_xstats_info {
2932  	struct netdev_notifier_info info; /* must be first */
2933  	enum netdev_offload_xstats_type type;
2934  
2935  	union {
2936  		/* NETDEV_OFFLOAD_XSTATS_REPORT_DELTA */
2937  		struct netdev_notifier_offload_xstats_rd *report_delta;
2938  		/* NETDEV_OFFLOAD_XSTATS_REPORT_USED */
2939  		struct netdev_notifier_offload_xstats_ru *report_used;
2940  	};
2941  };
2942  
2943  int netdev_offload_xstats_enable(struct net_device *dev,
2944  				 enum netdev_offload_xstats_type type,
2945  				 struct netlink_ext_ack *extack);
2946  int netdev_offload_xstats_disable(struct net_device *dev,
2947  				  enum netdev_offload_xstats_type type);
2948  bool netdev_offload_xstats_enabled(const struct net_device *dev,
2949  				   enum netdev_offload_xstats_type type);
2950  int netdev_offload_xstats_get(struct net_device *dev,
2951  			      enum netdev_offload_xstats_type type,
2952  			      struct rtnl_hw_stats64 *stats, bool *used,
2953  			      struct netlink_ext_ack *extack);
2954  void
2955  netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *rd,
2956  				   const struct rtnl_hw_stats64 *stats);
2957  void
2958  netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *ru);
2959  void netdev_offload_xstats_push_delta(struct net_device *dev,
2960  				      enum netdev_offload_xstats_type type,
2961  				      const struct rtnl_hw_stats64 *stats);
2962  
netdev_notifier_info_init(struct netdev_notifier_info * info,struct net_device * dev)2963  static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2964  					     struct net_device *dev)
2965  {
2966  	info->dev = dev;
2967  	info->extack = NULL;
2968  }
2969  
2970  static inline struct net_device *
netdev_notifier_info_to_dev(const struct netdev_notifier_info * info)2971  netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2972  {
2973  	return info->dev;
2974  }
2975  
2976  static inline struct netlink_ext_ack *
netdev_notifier_info_to_extack(const struct netdev_notifier_info * info)2977  netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
2978  {
2979  	return info->extack;
2980  }
2981  
2982  int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2983  
2984  
2985  extern rwlock_t				dev_base_lock;		/* Device list lock */
2986  
2987  #define for_each_netdev(net, d)		\
2988  		list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2989  #define for_each_netdev_reverse(net, d)	\
2990  		list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2991  #define for_each_netdev_rcu(net, d)		\
2992  		list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2993  #define for_each_netdev_safe(net, d, n)	\
2994  		list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2995  #define for_each_netdev_continue(net, d)		\
2996  		list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2997  #define for_each_netdev_continue_reverse(net, d)		\
2998  		list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
2999  						     dev_list)
3000  #define for_each_netdev_continue_rcu(net, d)		\
3001  	list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
3002  #define for_each_netdev_in_bond_rcu(bond, slave)	\
3003  		for_each_netdev_rcu(&init_net, slave)	\
3004  			if (netdev_master_upper_dev_get_rcu(slave) == (bond))
3005  #define net_device_entry(lh)	list_entry(lh, struct net_device, dev_list)
3006  
next_net_device(struct net_device * dev)3007  static inline struct net_device *next_net_device(struct net_device *dev)
3008  {
3009  	struct list_head *lh;
3010  	struct net *net;
3011  
3012  	net = dev_net(dev);
3013  	lh = dev->dev_list.next;
3014  	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3015  }
3016  
next_net_device_rcu(struct net_device * dev)3017  static inline struct net_device *next_net_device_rcu(struct net_device *dev)
3018  {
3019  	struct list_head *lh;
3020  	struct net *net;
3021  
3022  	net = dev_net(dev);
3023  	lh = rcu_dereference(list_next_rcu(&dev->dev_list));
3024  	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3025  }
3026  
first_net_device(struct net * net)3027  static inline struct net_device *first_net_device(struct net *net)
3028  {
3029  	return list_empty(&net->dev_base_head) ? NULL :
3030  		net_device_entry(net->dev_base_head.next);
3031  }
3032  
first_net_device_rcu(struct net * net)3033  static inline struct net_device *first_net_device_rcu(struct net *net)
3034  {
3035  	struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
3036  
3037  	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3038  }
3039  
3040  int netdev_boot_setup_check(struct net_device *dev);
3041  struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
3042  				       const char *hwaddr);
3043  struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
3044  void dev_add_pack(struct packet_type *pt);
3045  void dev_remove_pack(struct packet_type *pt);
3046  void __dev_remove_pack(struct packet_type *pt);
3047  void dev_add_offload(struct packet_offload *po);
3048  void dev_remove_offload(struct packet_offload *po);
3049  
3050  int dev_get_iflink(const struct net_device *dev);
3051  int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
3052  int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
3053  			  struct net_device_path_stack *stack);
3054  struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
3055  				      unsigned short mask);
3056  struct net_device *dev_get_by_name(struct net *net, const char *name);
3057  struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
3058  struct net_device *__dev_get_by_name(struct net *net, const char *name);
3059  bool netdev_name_in_use(struct net *net, const char *name);
3060  int dev_alloc_name(struct net_device *dev, const char *name);
3061  int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
3062  void dev_close(struct net_device *dev);
3063  void dev_close_many(struct list_head *head, bool unlink);
3064  void dev_disable_lro(struct net_device *dev);
3065  int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
3066  u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
3067  		     struct net_device *sb_dev);
3068  u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
3069  		       struct net_device *sb_dev);
3070  
3071  int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev);
3072  int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
3073  
dev_queue_xmit(struct sk_buff * skb)3074  static inline int dev_queue_xmit(struct sk_buff *skb)
3075  {
3076  	return __dev_queue_xmit(skb, NULL);
3077  }
3078  
dev_queue_xmit_accel(struct sk_buff * skb,struct net_device * sb_dev)3079  static inline int dev_queue_xmit_accel(struct sk_buff *skb,
3080  				       struct net_device *sb_dev)
3081  {
3082  	return __dev_queue_xmit(skb, sb_dev);
3083  }
3084  
dev_direct_xmit(struct sk_buff * skb,u16 queue_id)3085  static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3086  {
3087  	int ret;
3088  
3089  	ret = __dev_direct_xmit(skb, queue_id);
3090  	if (!dev_xmit_complete(ret))
3091  		kfree_skb(skb);
3092  	return ret;
3093  }
3094  
3095  int register_netdevice(struct net_device *dev);
3096  void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
3097  void unregister_netdevice_many(struct list_head *head);
unregister_netdevice(struct net_device * dev)3098  static inline void unregister_netdevice(struct net_device *dev)
3099  {
3100  	unregister_netdevice_queue(dev, NULL);
3101  }
3102  
3103  int netdev_refcnt_read(const struct net_device *dev);
3104  void free_netdev(struct net_device *dev);
3105  void netdev_freemem(struct net_device *dev);
3106  int init_dummy_netdev(struct net_device *dev);
3107  
3108  struct net_device *netdev_get_xmit_slave(struct net_device *dev,
3109  					 struct sk_buff *skb,
3110  					 bool all_slaves);
3111  struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
3112  					    struct sock *sk);
3113  struct net_device *dev_get_by_index(struct net *net, int ifindex);
3114  struct net_device *__dev_get_by_index(struct net *net, int ifindex);
3115  struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
3116  struct net_device *dev_get_by_napi_id(unsigned int napi_id);
3117  int dev_restart(struct net_device *dev);
3118  
3119  
dev_hard_header(struct sk_buff * skb,struct net_device * dev,unsigned short type,const void * daddr,const void * saddr,unsigned int len)3120  static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
3121  				  unsigned short type,
3122  				  const void *daddr, const void *saddr,
3123  				  unsigned int len)
3124  {
3125  	if (!dev->header_ops || !dev->header_ops->create)
3126  		return 0;
3127  
3128  	return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
3129  }
3130  
dev_parse_header(const struct sk_buff * skb,unsigned char * haddr)3131  static inline int dev_parse_header(const struct sk_buff *skb,
3132  				   unsigned char *haddr)
3133  {
3134  	const struct net_device *dev = skb->dev;
3135  
3136  	if (!dev->header_ops || !dev->header_ops->parse)
3137  		return 0;
3138  	return dev->header_ops->parse(skb, haddr);
3139  }
3140  
dev_parse_header_protocol(const struct sk_buff * skb)3141  static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
3142  {
3143  	const struct net_device *dev = skb->dev;
3144  
3145  	if (!dev->header_ops || !dev->header_ops->parse_protocol)
3146  		return 0;
3147  	return dev->header_ops->parse_protocol(skb);
3148  }
3149  
3150  /* ll_header must have at least hard_header_len allocated */
dev_validate_header(const struct net_device * dev,char * ll_header,int len)3151  static inline bool dev_validate_header(const struct net_device *dev,
3152  				       char *ll_header, int len)
3153  {
3154  	if (likely(len >= dev->hard_header_len))
3155  		return true;
3156  	if (len < dev->min_header_len)
3157  		return false;
3158  
3159  	if (capable(CAP_SYS_RAWIO)) {
3160  		memset(ll_header + len, 0, dev->hard_header_len - len);
3161  		return true;
3162  	}
3163  
3164  	if (dev->header_ops && dev->header_ops->validate)
3165  		return dev->header_ops->validate(ll_header, len);
3166  
3167  	return false;
3168  }
3169  
dev_has_header(const struct net_device * dev)3170  static inline bool dev_has_header(const struct net_device *dev)
3171  {
3172  	return dev->header_ops && dev->header_ops->create;
3173  }
3174  
3175  /*
3176   * Incoming packets are placed on per-CPU queues
3177   */
3178  struct softnet_data {
3179  	struct list_head	poll_list;
3180  	struct sk_buff_head	process_queue;
3181  
3182  	/* stats */
3183  	unsigned int		processed;
3184  	unsigned int		time_squeeze;
3185  	unsigned int		received_rps;
3186  #ifdef CONFIG_RPS
3187  	struct softnet_data	*rps_ipi_list;
3188  #endif
3189  #ifdef CONFIG_NET_FLOW_LIMIT
3190  	struct sd_flow_limit __rcu *flow_limit;
3191  #endif
3192  	struct Qdisc		*output_queue;
3193  	struct Qdisc		**output_queue_tailp;
3194  	struct sk_buff		*completion_queue;
3195  #ifdef CONFIG_XFRM_OFFLOAD
3196  	struct sk_buff_head	xfrm_backlog;
3197  #endif
3198  	/* written and read only by owning cpu: */
3199  	struct {
3200  		u16 recursion;
3201  		u8  more;
3202  #ifdef CONFIG_NET_EGRESS
3203  		u8  skip_txqueue;
3204  #endif
3205  	} xmit;
3206  #ifdef CONFIG_RPS
3207  	/* input_queue_head should be written by cpu owning this struct,
3208  	 * and only read by other cpus. Worth using a cache line.
3209  	 */
3210  	unsigned int		input_queue_head ____cacheline_aligned_in_smp;
3211  
3212  	/* Elements below can be accessed between CPUs for RPS/RFS */
3213  	call_single_data_t	csd ____cacheline_aligned_in_smp;
3214  	struct softnet_data	*rps_ipi_next;
3215  	unsigned int		cpu;
3216  	unsigned int		input_queue_tail;
3217  #endif
3218  	unsigned int		dropped;
3219  	struct sk_buff_head	input_pkt_queue;
3220  	struct napi_struct	backlog;
3221  
3222  	/* Another possibly contended cache line */
3223  	spinlock_t		defer_lock ____cacheline_aligned_in_smp;
3224  	int			defer_count;
3225  	int			defer_ipi_scheduled;
3226  	struct sk_buff		*defer_list;
3227  	call_single_data_t	defer_csd;
3228  };
3229  
input_queue_head_incr(struct softnet_data * sd)3230  static inline void input_queue_head_incr(struct softnet_data *sd)
3231  {
3232  #ifdef CONFIG_RPS
3233  	sd->input_queue_head++;
3234  #endif
3235  }
3236  
input_queue_tail_incr_save(struct softnet_data * sd,unsigned int * qtail)3237  static inline void input_queue_tail_incr_save(struct softnet_data *sd,
3238  					      unsigned int *qtail)
3239  {
3240  #ifdef CONFIG_RPS
3241  	*qtail = ++sd->input_queue_tail;
3242  #endif
3243  }
3244  
3245  DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
3246  
dev_recursion_level(void)3247  static inline int dev_recursion_level(void)
3248  {
3249  	return this_cpu_read(softnet_data.xmit.recursion);
3250  }
3251  
3252  #define XMIT_RECURSION_LIMIT	8
dev_xmit_recursion(void)3253  static inline bool dev_xmit_recursion(void)
3254  {
3255  	return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
3256  			XMIT_RECURSION_LIMIT);
3257  }
3258  
dev_xmit_recursion_inc(void)3259  static inline void dev_xmit_recursion_inc(void)
3260  {
3261  	__this_cpu_inc(softnet_data.xmit.recursion);
3262  }
3263  
dev_xmit_recursion_dec(void)3264  static inline void dev_xmit_recursion_dec(void)
3265  {
3266  	__this_cpu_dec(softnet_data.xmit.recursion);
3267  }
3268  
3269  void __netif_schedule(struct Qdisc *q);
3270  void netif_schedule_queue(struct netdev_queue *txq);
3271  
netif_tx_schedule_all(struct net_device * dev)3272  static inline void netif_tx_schedule_all(struct net_device *dev)
3273  {
3274  	unsigned int i;
3275  
3276  	for (i = 0; i < dev->num_tx_queues; i++)
3277  		netif_schedule_queue(netdev_get_tx_queue(dev, i));
3278  }
3279  
netif_tx_start_queue(struct netdev_queue * dev_queue)3280  static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
3281  {
3282  	clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3283  }
3284  
3285  /**
3286   *	netif_start_queue - allow transmit
3287   *	@dev: network device
3288   *
3289   *	Allow upper layers to call the device hard_start_xmit routine.
3290   */
netif_start_queue(struct net_device * dev)3291  static inline void netif_start_queue(struct net_device *dev)
3292  {
3293  	netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
3294  }
3295  
netif_tx_start_all_queues(struct net_device * dev)3296  static inline void netif_tx_start_all_queues(struct net_device *dev)
3297  {
3298  	unsigned int i;
3299  
3300  	for (i = 0; i < dev->num_tx_queues; i++) {
3301  		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3302  		netif_tx_start_queue(txq);
3303  	}
3304  }
3305  
3306  void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3307  
3308  /**
3309   *	netif_wake_queue - restart transmit
3310   *	@dev: network device
3311   *
3312   *	Allow upper layers to call the device hard_start_xmit routine.
3313   *	Used for flow control when transmit resources are available.
3314   */
netif_wake_queue(struct net_device * dev)3315  static inline void netif_wake_queue(struct net_device *dev)
3316  {
3317  	netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
3318  }
3319  
netif_tx_wake_all_queues(struct net_device * dev)3320  static inline void netif_tx_wake_all_queues(struct net_device *dev)
3321  {
3322  	unsigned int i;
3323  
3324  	for (i = 0; i < dev->num_tx_queues; i++) {
3325  		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3326  		netif_tx_wake_queue(txq);
3327  	}
3328  }
3329  
netif_tx_stop_queue(struct netdev_queue * dev_queue)3330  static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
3331  {
3332  	set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3333  }
3334  
3335  /**
3336   *	netif_stop_queue - stop transmitted packets
3337   *	@dev: network device
3338   *
3339   *	Stop upper layers calling the device hard_start_xmit routine.
3340   *	Used for flow control when transmit resources are unavailable.
3341   */
netif_stop_queue(struct net_device * dev)3342  static inline void netif_stop_queue(struct net_device *dev)
3343  {
3344  	netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
3345  }
3346  
3347  void netif_tx_stop_all_queues(struct net_device *dev);
3348  
netif_tx_queue_stopped(const struct netdev_queue * dev_queue)3349  static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
3350  {
3351  	return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3352  }
3353  
3354  /**
3355   *	netif_queue_stopped - test if transmit queue is flowblocked
3356   *	@dev: network device
3357   *
3358   *	Test if transmit queue on device is currently unable to send.
3359   */
netif_queue_stopped(const struct net_device * dev)3360  static inline bool netif_queue_stopped(const struct net_device *dev)
3361  {
3362  	return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
3363  }
3364  
netif_xmit_stopped(const struct netdev_queue * dev_queue)3365  static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
3366  {
3367  	return dev_queue->state & QUEUE_STATE_ANY_XOFF;
3368  }
3369  
3370  static inline bool
netif_xmit_frozen_or_stopped(const struct netdev_queue * dev_queue)3371  netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
3372  {
3373  	return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
3374  }
3375  
3376  static inline bool
netif_xmit_frozen_or_drv_stopped(const struct netdev_queue * dev_queue)3377  netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
3378  {
3379  	return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
3380  }
3381  
3382  /**
3383   *	netdev_queue_set_dql_min_limit - set dql minimum limit
3384   *	@dev_queue: pointer to transmit queue
3385   *	@min_limit: dql minimum limit
3386   *
3387   * Forces xmit_more() to return true until the minimum threshold
3388   * defined by @min_limit is reached (or until the tx queue is
3389   * empty). Warning: to be use with care, misuse will impact the
3390   * latency.
3391   */
netdev_queue_set_dql_min_limit(struct netdev_queue * dev_queue,unsigned int min_limit)3392  static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,
3393  						  unsigned int min_limit)
3394  {
3395  #ifdef CONFIG_BQL
3396  	dev_queue->dql.min_limit = min_limit;
3397  #endif
3398  }
3399  
3400  /**
3401   *	netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
3402   *	@dev_queue: pointer to transmit queue
3403   *
3404   * BQL enabled drivers might use this helper in their ndo_start_xmit(),
3405   * to give appropriate hint to the CPU.
3406   */
netdev_txq_bql_enqueue_prefetchw(struct netdev_queue * dev_queue)3407  static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
3408  {
3409  #ifdef CONFIG_BQL
3410  	prefetchw(&dev_queue->dql.num_queued);
3411  #endif
3412  }
3413  
3414  /**
3415   *	netdev_txq_bql_complete_prefetchw - prefetch bql data for write
3416   *	@dev_queue: pointer to transmit queue
3417   *
3418   * BQL enabled drivers might use this helper in their TX completion path,
3419   * to give appropriate hint to the CPU.
3420   */
netdev_txq_bql_complete_prefetchw(struct netdev_queue * dev_queue)3421  static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3422  {
3423  #ifdef CONFIG_BQL
3424  	prefetchw(&dev_queue->dql.limit);
3425  #endif
3426  }
3427  
3428  /**
3429   *	netdev_tx_sent_queue - report the number of bytes queued to a given tx queue
3430   *	@dev_queue: network device queue
3431   *	@bytes: number of bytes queued to the device queue
3432   *
3433   *	Report the number of bytes queued for sending/completion to the network
3434   *	device hardware queue. @bytes should be a good approximation and should
3435   *	exactly match netdev_completed_queue() @bytes.
3436   *	This is typically called once per packet, from ndo_start_xmit().
3437   */
netdev_tx_sent_queue(struct netdev_queue * dev_queue,unsigned int bytes)3438  static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3439  					unsigned int bytes)
3440  {
3441  #ifdef CONFIG_BQL
3442  	dql_queued(&dev_queue->dql, bytes);
3443  
3444  	if (likely(dql_avail(&dev_queue->dql) >= 0))
3445  		return;
3446  
3447  	set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3448  
3449  	/*
3450  	 * The XOFF flag must be set before checking the dql_avail below,
3451  	 * because in netdev_tx_completed_queue we update the dql_completed
3452  	 * before checking the XOFF flag.
3453  	 */
3454  	smp_mb();
3455  
3456  	/* check again in case another CPU has just made room avail */
3457  	if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3458  		clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3459  #endif
3460  }
3461  
3462  /* Variant of netdev_tx_sent_queue() for drivers that are aware
3463   * that they should not test BQL status themselves.
3464   * We do want to change __QUEUE_STATE_STACK_XOFF only for the last
3465   * skb of a batch.
3466   * Returns true if the doorbell must be used to kick the NIC.
3467   */
__netdev_tx_sent_queue(struct netdev_queue * dev_queue,unsigned int bytes,bool xmit_more)3468  static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3469  					  unsigned int bytes,
3470  					  bool xmit_more)
3471  {
3472  	if (xmit_more) {
3473  #ifdef CONFIG_BQL
3474  		dql_queued(&dev_queue->dql, bytes);
3475  #endif
3476  		return netif_tx_queue_stopped(dev_queue);
3477  	}
3478  	netdev_tx_sent_queue(dev_queue, bytes);
3479  	return true;
3480  }
3481  
3482  /**
3483   *	netdev_sent_queue - report the number of bytes queued to hardware
3484   *	@dev: network device
3485   *	@bytes: number of bytes queued to the hardware device queue
3486   *
3487   *	Report the number of bytes queued for sending/completion to the network
3488   *	device hardware queue#0. @bytes should be a good approximation and should
3489   *	exactly match netdev_completed_queue() @bytes.
3490   *	This is typically called once per packet, from ndo_start_xmit().
3491   */
netdev_sent_queue(struct net_device * dev,unsigned int bytes)3492  static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3493  {
3494  	netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3495  }
3496  
__netdev_sent_queue(struct net_device * dev,unsigned int bytes,bool xmit_more)3497  static inline bool __netdev_sent_queue(struct net_device *dev,
3498  				       unsigned int bytes,
3499  				       bool xmit_more)
3500  {
3501  	return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
3502  				      xmit_more);
3503  }
3504  
3505  /**
3506   *	netdev_tx_completed_queue - report number of packets/bytes at TX completion.
3507   *	@dev_queue: network device queue
3508   *	@pkts: number of packets (currently ignored)
3509   *	@bytes: number of bytes dequeued from the device queue
3510   *
3511   *	Must be called at most once per TX completion round (and not per
3512   *	individual packet), so that BQL can adjust its limits appropriately.
3513   */
netdev_tx_completed_queue(struct netdev_queue * dev_queue,unsigned int pkts,unsigned int bytes)3514  static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3515  					     unsigned int pkts, unsigned int bytes)
3516  {
3517  #ifdef CONFIG_BQL
3518  	if (unlikely(!bytes))
3519  		return;
3520  
3521  	dql_completed(&dev_queue->dql, bytes);
3522  
3523  	/*
3524  	 * Without the memory barrier there is a small possiblity that
3525  	 * netdev_tx_sent_queue will miss the update and cause the queue to
3526  	 * be stopped forever
3527  	 */
3528  	smp_mb();
3529  
3530  	if (unlikely(dql_avail(&dev_queue->dql) < 0))
3531  		return;
3532  
3533  	if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3534  		netif_schedule_queue(dev_queue);
3535  #endif
3536  }
3537  
3538  /**
3539   * 	netdev_completed_queue - report bytes and packets completed by device
3540   * 	@dev: network device
3541   * 	@pkts: actual number of packets sent over the medium
3542   * 	@bytes: actual number of bytes sent over the medium
3543   *
3544   * 	Report the number of bytes and packets transmitted by the network device
3545   * 	hardware queue over the physical medium, @bytes must exactly match the
3546   * 	@bytes amount passed to netdev_sent_queue()
3547   */
netdev_completed_queue(struct net_device * dev,unsigned int pkts,unsigned int bytes)3548  static inline void netdev_completed_queue(struct net_device *dev,
3549  					  unsigned int pkts, unsigned int bytes)
3550  {
3551  	netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3552  }
3553  
netdev_tx_reset_queue(struct netdev_queue * q)3554  static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3555  {
3556  #ifdef CONFIG_BQL
3557  	clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3558  	dql_reset(&q->dql);
3559  #endif
3560  }
3561  
3562  /**
3563   * 	netdev_reset_queue - reset the packets and bytes count of a network device
3564   * 	@dev_queue: network device
3565   *
3566   * 	Reset the bytes and packet count of a network device and clear the
3567   * 	software flow control OFF bit for this network device
3568   */
netdev_reset_queue(struct net_device * dev_queue)3569  static inline void netdev_reset_queue(struct net_device *dev_queue)
3570  {
3571  	netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
3572  }
3573  
3574  /**
3575   * 	netdev_cap_txqueue - check if selected tx queue exceeds device queues
3576   * 	@dev: network device
3577   * 	@queue_index: given tx queue index
3578   *
3579   * 	Returns 0 if given tx queue index >= number of device tx queues,
3580   * 	otherwise returns the originally passed tx queue index.
3581   */
netdev_cap_txqueue(struct net_device * dev,u16 queue_index)3582  static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3583  {
3584  	if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3585  		net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3586  				     dev->name, queue_index,
3587  				     dev->real_num_tx_queues);
3588  		return 0;
3589  	}
3590  
3591  	return queue_index;
3592  }
3593  
3594  /**
3595   *	netif_running - test if up
3596   *	@dev: network device
3597   *
3598   *	Test if the device has been brought up.
3599   */
netif_running(const struct net_device * dev)3600  static inline bool netif_running(const struct net_device *dev)
3601  {
3602  	return test_bit(__LINK_STATE_START, &dev->state);
3603  }
3604  
3605  /*
3606   * Routines to manage the subqueues on a device.  We only need start,
3607   * stop, and a check if it's stopped.  All other device management is
3608   * done at the overall netdevice level.
3609   * Also test the device if we're multiqueue.
3610   */
3611  
3612  /**
3613   *	netif_start_subqueue - allow sending packets on subqueue
3614   *	@dev: network device
3615   *	@queue_index: sub queue index
3616   *
3617   * Start individual transmit queue of a device with multiple transmit queues.
3618   */
netif_start_subqueue(struct net_device * dev,u16 queue_index)3619  static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
3620  {
3621  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3622  
3623  	netif_tx_start_queue(txq);
3624  }
3625  
3626  /**
3627   *	netif_stop_subqueue - stop sending packets on subqueue
3628   *	@dev: network device
3629   *	@queue_index: sub queue index
3630   *
3631   * Stop individual transmit queue of a device with multiple transmit queues.
3632   */
netif_stop_subqueue(struct net_device * dev,u16 queue_index)3633  static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
3634  {
3635  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3636  	netif_tx_stop_queue(txq);
3637  }
3638  
3639  /**
3640   *	__netif_subqueue_stopped - test status of subqueue
3641   *	@dev: network device
3642   *	@queue_index: sub queue index
3643   *
3644   * Check individual transmit queue of a device with multiple transmit queues.
3645   */
__netif_subqueue_stopped(const struct net_device * dev,u16 queue_index)3646  static inline bool __netif_subqueue_stopped(const struct net_device *dev,
3647  					    u16 queue_index)
3648  {
3649  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3650  
3651  	return netif_tx_queue_stopped(txq);
3652  }
3653  
3654  /**
3655   *	netif_subqueue_stopped - test status of subqueue
3656   *	@dev: network device
3657   *	@skb: sub queue buffer pointer
3658   *
3659   * Check individual transmit queue of a device with multiple transmit queues.
3660   */
netif_subqueue_stopped(const struct net_device * dev,struct sk_buff * skb)3661  static inline bool netif_subqueue_stopped(const struct net_device *dev,
3662  					  struct sk_buff *skb)
3663  {
3664  	return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
3665  }
3666  
3667  /**
3668   *	netif_wake_subqueue - allow sending packets on subqueue
3669   *	@dev: network device
3670   *	@queue_index: sub queue index
3671   *
3672   * Resume individual transmit queue of a device with multiple transmit queues.
3673   */
netif_wake_subqueue(struct net_device * dev,u16 queue_index)3674  static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
3675  {
3676  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3677  
3678  	netif_tx_wake_queue(txq);
3679  }
3680  
3681  #ifdef CONFIG_XPS
3682  int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3683  			u16 index);
3684  int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
3685  			  u16 index, enum xps_map_type type);
3686  
3687  /**
3688   *	netif_attr_test_mask - Test a CPU or Rx queue set in a mask
3689   *	@j: CPU/Rx queue index
3690   *	@mask: bitmask of all cpus/rx queues
3691   *	@nr_bits: number of bits in the bitmask
3692   *
3693   * Test if a CPU or Rx queue index is set in a mask of all CPU/Rx queues.
3694   */
netif_attr_test_mask(unsigned long j,const unsigned long * mask,unsigned int nr_bits)3695  static inline bool netif_attr_test_mask(unsigned long j,
3696  					const unsigned long *mask,
3697  					unsigned int nr_bits)
3698  {
3699  	cpu_max_bits_warn(j, nr_bits);
3700  	return test_bit(j, mask);
3701  }
3702  
3703  /**
3704   *	netif_attr_test_online - Test for online CPU/Rx queue
3705   *	@j: CPU/Rx queue index
3706   *	@online_mask: bitmask for CPUs/Rx queues that are online
3707   *	@nr_bits: number of bits in the bitmask
3708   *
3709   * Returns true if a CPU/Rx queue is online.
3710   */
netif_attr_test_online(unsigned long j,const unsigned long * online_mask,unsigned int nr_bits)3711  static inline bool netif_attr_test_online(unsigned long j,
3712  					  const unsigned long *online_mask,
3713  					  unsigned int nr_bits)
3714  {
3715  	cpu_max_bits_warn(j, nr_bits);
3716  
3717  	if (online_mask)
3718  		return test_bit(j, online_mask);
3719  
3720  	return (j < nr_bits);
3721  }
3722  
3723  /**
3724   *	netif_attrmask_next - get the next CPU/Rx queue in a cpu/Rx queues mask
3725   *	@n: CPU/Rx queue index
3726   *	@srcp: the cpumask/Rx queue mask pointer
3727   *	@nr_bits: number of bits in the bitmask
3728   *
3729   * Returns >= nr_bits if no further CPUs/Rx queues set.
3730   */
netif_attrmask_next(int n,const unsigned long * srcp,unsigned int nr_bits)3731  static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
3732  					       unsigned int nr_bits)
3733  {
3734  	/* -1 is a legal arg here. */
3735  	if (n != -1)
3736  		cpu_max_bits_warn(n, nr_bits);
3737  
3738  	if (srcp)
3739  		return find_next_bit(srcp, nr_bits, n + 1);
3740  
3741  	return n + 1;
3742  }
3743  
3744  /**
3745   *	netif_attrmask_next_and - get the next CPU/Rx queue in \*src1p & \*src2p
3746   *	@n: CPU/Rx queue index
3747   *	@src1p: the first CPUs/Rx queues mask pointer
3748   *	@src2p: the second CPUs/Rx queues mask pointer
3749   *	@nr_bits: number of bits in the bitmask
3750   *
3751   * Returns >= nr_bits if no further CPUs/Rx queues set in both.
3752   */
netif_attrmask_next_and(int n,const unsigned long * src1p,const unsigned long * src2p,unsigned int nr_bits)3753  static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
3754  					  const unsigned long *src2p,
3755  					  unsigned int nr_bits)
3756  {
3757  	/* -1 is a legal arg here. */
3758  	if (n != -1)
3759  		cpu_max_bits_warn(n, nr_bits);
3760  
3761  	if (src1p && src2p)
3762  		return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
3763  	else if (src1p)
3764  		return find_next_bit(src1p, nr_bits, n + 1);
3765  	else if (src2p)
3766  		return find_next_bit(src2p, nr_bits, n + 1);
3767  
3768  	return n + 1;
3769  }
3770  #else
netif_set_xps_queue(struct net_device * dev,const struct cpumask * mask,u16 index)3771  static inline int netif_set_xps_queue(struct net_device *dev,
3772  				      const struct cpumask *mask,
3773  				      u16 index)
3774  {
3775  	return 0;
3776  }
3777  
__netif_set_xps_queue(struct net_device * dev,const unsigned long * mask,u16 index,enum xps_map_type type)3778  static inline int __netif_set_xps_queue(struct net_device *dev,
3779  					const unsigned long *mask,
3780  					u16 index, enum xps_map_type type)
3781  {
3782  	return 0;
3783  }
3784  #endif
3785  
3786  /**
3787   *	netif_is_multiqueue - test if device has multiple transmit queues
3788   *	@dev: network device
3789   *
3790   * Check if device has multiple transmit queues
3791   */
netif_is_multiqueue(const struct net_device * dev)3792  static inline bool netif_is_multiqueue(const struct net_device *dev)
3793  {
3794  	return dev->num_tx_queues > 1;
3795  }
3796  
3797  int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
3798  
3799  #ifdef CONFIG_SYSFS
3800  int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
3801  #else
netif_set_real_num_rx_queues(struct net_device * dev,unsigned int rxqs)3802  static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3803  						unsigned int rxqs)
3804  {
3805  	dev->real_num_rx_queues = rxqs;
3806  	return 0;
3807  }
3808  #endif
3809  int netif_set_real_num_queues(struct net_device *dev,
3810  			      unsigned int txq, unsigned int rxq);
3811  
3812  static inline struct netdev_rx_queue *
__netif_get_rx_queue(struct net_device * dev,unsigned int rxq)3813  __netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
3814  {
3815  	return dev->_rx + rxq;
3816  }
3817  
3818  #ifdef CONFIG_SYSFS
get_netdev_rx_queue_index(struct netdev_rx_queue * queue)3819  static inline unsigned int get_netdev_rx_queue_index(
3820  		struct netdev_rx_queue *queue)
3821  {
3822  	struct net_device *dev = queue->dev;
3823  	int index = queue - dev->_rx;
3824  
3825  	BUG_ON(index >= dev->num_rx_queues);
3826  	return index;
3827  }
3828  #endif
3829  
3830  int netif_get_num_default_rss_queues(void);
3831  
3832  enum skb_free_reason {
3833  	SKB_REASON_CONSUMED,
3834  	SKB_REASON_DROPPED,
3835  };
3836  
3837  void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
3838  void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
3839  
3840  /*
3841   * It is not allowed to call kfree_skb() or consume_skb() from hardware
3842   * interrupt context or with hardware interrupts being disabled.
3843   * (in_hardirq() || irqs_disabled())
3844   *
3845   * We provide four helpers that can be used in following contexts :
3846   *
3847   * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
3848   *  replacing kfree_skb(skb)
3849   *
3850   * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
3851   *  Typically used in place of consume_skb(skb) in TX completion path
3852   *
3853   * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
3854   *  replacing kfree_skb(skb)
3855   *
3856   * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
3857   *  and consumed a packet. Used in place of consume_skb(skb)
3858   */
dev_kfree_skb_irq(struct sk_buff * skb)3859  static inline void dev_kfree_skb_irq(struct sk_buff *skb)
3860  {
3861  	__dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
3862  }
3863  
dev_consume_skb_irq(struct sk_buff * skb)3864  static inline void dev_consume_skb_irq(struct sk_buff *skb)
3865  {
3866  	__dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
3867  }
3868  
dev_kfree_skb_any(struct sk_buff * skb)3869  static inline void dev_kfree_skb_any(struct sk_buff *skb)
3870  {
3871  	__dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
3872  }
3873  
dev_consume_skb_any(struct sk_buff * skb)3874  static inline void dev_consume_skb_any(struct sk_buff *skb)
3875  {
3876  	__dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
3877  }
3878  
3879  u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
3880  			     struct bpf_prog *xdp_prog);
3881  void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
3882  int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
3883  int netif_rx(struct sk_buff *skb);
3884  int __netif_rx(struct sk_buff *skb);
3885  
3886  int netif_receive_skb(struct sk_buff *skb);
3887  int netif_receive_skb_core(struct sk_buff *skb);
3888  void netif_receive_skb_list_internal(struct list_head *head);
3889  void netif_receive_skb_list(struct list_head *head);
3890  gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3891  void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3892  struct sk_buff *napi_get_frags(struct napi_struct *napi);
3893  void napi_get_frags_check(struct napi_struct *napi);
3894  gro_result_t napi_gro_frags(struct napi_struct *napi);
3895  struct packet_offload *gro_find_receive_by_type(__be16 type);
3896  struct packet_offload *gro_find_complete_by_type(__be16 type);
3897  
napi_free_frags(struct napi_struct * napi)3898  static inline void napi_free_frags(struct napi_struct *napi)
3899  {
3900  	kfree_skb(napi->skb);
3901  	napi->skb = NULL;
3902  }
3903  
3904  bool netdev_is_rx_handler_busy(struct net_device *dev);
3905  int netdev_rx_handler_register(struct net_device *dev,
3906  			       rx_handler_func_t *rx_handler,
3907  			       void *rx_handler_data);
3908  void netdev_rx_handler_unregister(struct net_device *dev);
3909  
3910  bool dev_valid_name(const char *name);
is_socket_ioctl_cmd(unsigned int cmd)3911  static inline bool is_socket_ioctl_cmd(unsigned int cmd)
3912  {
3913  	return _IOC_TYPE(cmd) == SOCK_IOC_TYPE;
3914  }
3915  int get_user_ifreq(struct ifreq *ifr, void __user **ifrdata, void __user *arg);
3916  int put_user_ifreq(struct ifreq *ifr, void __user *arg);
3917  int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
3918  		void __user *data, bool *need_copyout);
3919  int dev_ifconf(struct net *net, struct ifconf __user *ifc);
3920  int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
3921  unsigned int dev_get_flags(const struct net_device *);
3922  int __dev_change_flags(struct net_device *dev, unsigned int flags,
3923  		       struct netlink_ext_ack *extack);
3924  int dev_change_flags(struct net_device *dev, unsigned int flags,
3925  		     struct netlink_ext_ack *extack);
3926  void __dev_notify_flags(struct net_device *, unsigned int old_flags,
3927  			unsigned int gchanges);
3928  int dev_set_alias(struct net_device *, const char *, size_t);
3929  int dev_get_alias(const struct net_device *, char *, size_t);
3930  int __dev_change_net_namespace(struct net_device *dev, struct net *net,
3931  			       const char *pat, int new_ifindex);
3932  static inline
dev_change_net_namespace(struct net_device * dev,struct net * net,const char * pat)3933  int dev_change_net_namespace(struct net_device *dev, struct net *net,
3934  			     const char *pat)
3935  {
3936  	return __dev_change_net_namespace(dev, net, pat, 0);
3937  }
3938  int __dev_set_mtu(struct net_device *, int);
3939  int dev_set_mtu(struct net_device *, int);
3940  int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
3941  			      struct netlink_ext_ack *extack);
3942  int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
3943  			struct netlink_ext_ack *extack);
3944  int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
3945  			     struct netlink_ext_ack *extack);
3946  int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name);
3947  int dev_get_port_parent_id(struct net_device *dev,
3948  			   struct netdev_phys_item_id *ppid, bool recurse);
3949  bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
3950  struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
3951  struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3952  				    struct netdev_queue *txq, int *ret);
3953  
3954  int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
3955  u8 dev_xdp_prog_count(struct net_device *dev);
3956  u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
3957  
3958  int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3959  int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3960  int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
3961  bool is_skb_forwardable(const struct net_device *dev,
3962  			const struct sk_buff *skb);
3963  
__is_skb_forwardable(const struct net_device * dev,const struct sk_buff * skb,const bool check_mtu)3964  static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
3965  						 const struct sk_buff *skb,
3966  						 const bool check_mtu)
3967  {
3968  	const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
3969  	unsigned int len;
3970  
3971  	if (!(dev->flags & IFF_UP))
3972  		return false;
3973  
3974  	if (!check_mtu)
3975  		return true;
3976  
3977  	len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
3978  	if (skb->len <= len)
3979  		return true;
3980  
3981  	/* if TSO is enabled, we don't care about the length as the packet
3982  	 * could be forwarded without being segmented before
3983  	 */
3984  	if (skb_is_gso(skb))
3985  		return true;
3986  
3987  	return false;
3988  }
3989  
3990  struct net_device_core_stats __percpu *netdev_core_stats_alloc(struct net_device *dev);
3991  
dev_core_stats(struct net_device * dev)3992  static inline struct net_device_core_stats __percpu *dev_core_stats(struct net_device *dev)
3993  {
3994  	/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
3995  	struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
3996  
3997  	if (likely(p))
3998  		return p;
3999  
4000  	return netdev_core_stats_alloc(dev);
4001  }
4002  
4003  #define DEV_CORE_STATS_INC(FIELD)						\
4004  static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev)		\
4005  {										\
4006  	struct net_device_core_stats __percpu *p;				\
4007  										\
4008  	p = dev_core_stats(dev);						\
4009  	if (p)									\
4010  		this_cpu_inc(p->FIELD);						\
4011  }
4012  DEV_CORE_STATS_INC(rx_dropped)
DEV_CORE_STATS_INC(tx_dropped)4013  DEV_CORE_STATS_INC(tx_dropped)
4014  DEV_CORE_STATS_INC(rx_nohandler)
4015  DEV_CORE_STATS_INC(rx_otherhost_dropped)
4016  
4017  static __always_inline int ____dev_forward_skb(struct net_device *dev,
4018  					       struct sk_buff *skb,
4019  					       const bool check_mtu)
4020  {
4021  	if (skb_orphan_frags(skb, GFP_ATOMIC) ||
4022  	    unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
4023  		dev_core_stats_rx_dropped_inc(dev);
4024  		kfree_skb(skb);
4025  		return NET_RX_DROP;
4026  	}
4027  
4028  	skb_scrub_packet(skb, !net_eq(dev_net(dev), dev_net(skb->dev)));
4029  	skb->priority = 0;
4030  	return 0;
4031  }
4032  
4033  bool dev_nit_active(struct net_device *dev);
4034  void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
4035  
__dev_put(struct net_device * dev)4036  static inline void __dev_put(struct net_device *dev)
4037  {
4038  	if (dev) {
4039  #ifdef CONFIG_PCPU_DEV_REFCNT
4040  		this_cpu_dec(*dev->pcpu_refcnt);
4041  #else
4042  		refcount_dec(&dev->dev_refcnt);
4043  #endif
4044  	}
4045  }
4046  
__dev_hold(struct net_device * dev)4047  static inline void __dev_hold(struct net_device *dev)
4048  {
4049  	if (dev) {
4050  #ifdef CONFIG_PCPU_DEV_REFCNT
4051  		this_cpu_inc(*dev->pcpu_refcnt);
4052  #else
4053  		refcount_inc(&dev->dev_refcnt);
4054  #endif
4055  	}
4056  }
4057  
__netdev_tracker_alloc(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4058  static inline void __netdev_tracker_alloc(struct net_device *dev,
4059  					  netdevice_tracker *tracker,
4060  					  gfp_t gfp)
4061  {
4062  #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4063  	ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp);
4064  #endif
4065  }
4066  
4067  /* netdev_tracker_alloc() can upgrade a prior untracked reference
4068   * taken by dev_get_by_name()/dev_get_by_index() to a tracked one.
4069   */
netdev_tracker_alloc(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4070  static inline void netdev_tracker_alloc(struct net_device *dev,
4071  					netdevice_tracker *tracker, gfp_t gfp)
4072  {
4073  #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4074  	refcount_dec(&dev->refcnt_tracker.no_tracker);
4075  	__netdev_tracker_alloc(dev, tracker, gfp);
4076  #endif
4077  }
4078  
netdev_tracker_free(struct net_device * dev,netdevice_tracker * tracker)4079  static inline void netdev_tracker_free(struct net_device *dev,
4080  				       netdevice_tracker *tracker)
4081  {
4082  #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4083  	ref_tracker_free(&dev->refcnt_tracker, tracker);
4084  #endif
4085  }
4086  
netdev_hold(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4087  static inline void netdev_hold(struct net_device *dev,
4088  			       netdevice_tracker *tracker, gfp_t gfp)
4089  {
4090  	if (dev) {
4091  		__dev_hold(dev);
4092  		__netdev_tracker_alloc(dev, tracker, gfp);
4093  	}
4094  }
4095  
netdev_put(struct net_device * dev,netdevice_tracker * tracker)4096  static inline void netdev_put(struct net_device *dev,
4097  			      netdevice_tracker *tracker)
4098  {
4099  	if (dev) {
4100  		netdev_tracker_free(dev, tracker);
4101  		__dev_put(dev);
4102  	}
4103  }
4104  
4105  /**
4106   *	dev_hold - get reference to device
4107   *	@dev: network device
4108   *
4109   * Hold reference to device to keep it from being freed.
4110   * Try using netdev_hold() instead.
4111   */
dev_hold(struct net_device * dev)4112  static inline void dev_hold(struct net_device *dev)
4113  {
4114  	netdev_hold(dev, NULL, GFP_ATOMIC);
4115  }
4116  
4117  /**
4118   *	dev_put - release reference to device
4119   *	@dev: network device
4120   *
4121   * Release reference to device to allow it to be freed.
4122   * Try using netdev_put() instead.
4123   */
dev_put(struct net_device * dev)4124  static inline void dev_put(struct net_device *dev)
4125  {
4126  	netdev_put(dev, NULL);
4127  }
4128  
netdev_ref_replace(struct net_device * odev,struct net_device * ndev,netdevice_tracker * tracker,gfp_t gfp)4129  static inline void netdev_ref_replace(struct net_device *odev,
4130  				      struct net_device *ndev,
4131  				      netdevice_tracker *tracker,
4132  				      gfp_t gfp)
4133  {
4134  	if (odev)
4135  		netdev_tracker_free(odev, tracker);
4136  
4137  	__dev_hold(ndev);
4138  	__dev_put(odev);
4139  
4140  	if (ndev)
4141  		__netdev_tracker_alloc(ndev, tracker, gfp);
4142  }
4143  
4144  /* Carrier loss detection, dial on demand. The functions netif_carrier_on
4145   * and _off may be called from IRQ context, but it is caller
4146   * who is responsible for serialization of these calls.
4147   *
4148   * The name carrier is inappropriate, these functions should really be
4149   * called netif_lowerlayer_*() because they represent the state of any
4150   * kind of lower layer not just hardware media.
4151   */
4152  void linkwatch_fire_event(struct net_device *dev);
4153  
4154  /**
4155   *	netif_carrier_ok - test if carrier present
4156   *	@dev: network device
4157   *
4158   * Check if carrier is present on device
4159   */
netif_carrier_ok(const struct net_device * dev)4160  static inline bool netif_carrier_ok(const struct net_device *dev)
4161  {
4162  	return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
4163  }
4164  
4165  unsigned long dev_trans_start(struct net_device *dev);
4166  
4167  void __netdev_watchdog_up(struct net_device *dev);
4168  
4169  void netif_carrier_on(struct net_device *dev);
4170  void netif_carrier_off(struct net_device *dev);
4171  void netif_carrier_event(struct net_device *dev);
4172  
4173  /**
4174   *	netif_dormant_on - mark device as dormant.
4175   *	@dev: network device
4176   *
4177   * Mark device as dormant (as per RFC2863).
4178   *
4179   * The dormant state indicates that the relevant interface is not
4180   * actually in a condition to pass packets (i.e., it is not 'up') but is
4181   * in a "pending" state, waiting for some external event.  For "on-
4182   * demand" interfaces, this new state identifies the situation where the
4183   * interface is waiting for events to place it in the up state.
4184   */
netif_dormant_on(struct net_device * dev)4185  static inline void netif_dormant_on(struct net_device *dev)
4186  {
4187  	if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
4188  		linkwatch_fire_event(dev);
4189  }
4190  
4191  /**
4192   *	netif_dormant_off - set device as not dormant.
4193   *	@dev: network device
4194   *
4195   * Device is not in dormant state.
4196   */
netif_dormant_off(struct net_device * dev)4197  static inline void netif_dormant_off(struct net_device *dev)
4198  {
4199  	if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
4200  		linkwatch_fire_event(dev);
4201  }
4202  
4203  /**
4204   *	netif_dormant - test if device is dormant
4205   *	@dev: network device
4206   *
4207   * Check if device is dormant.
4208   */
netif_dormant(const struct net_device * dev)4209  static inline bool netif_dormant(const struct net_device *dev)
4210  {
4211  	return test_bit(__LINK_STATE_DORMANT, &dev->state);
4212  }
4213  
4214  
4215  /**
4216   *	netif_testing_on - mark device as under test.
4217   *	@dev: network device
4218   *
4219   * Mark device as under test (as per RFC2863).
4220   *
4221   * The testing state indicates that some test(s) must be performed on
4222   * the interface. After completion, of the test, the interface state
4223   * will change to up, dormant, or down, as appropriate.
4224   */
netif_testing_on(struct net_device * dev)4225  static inline void netif_testing_on(struct net_device *dev)
4226  {
4227  	if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
4228  		linkwatch_fire_event(dev);
4229  }
4230  
4231  /**
4232   *	netif_testing_off - set device as not under test.
4233   *	@dev: network device
4234   *
4235   * Device is not in testing state.
4236   */
netif_testing_off(struct net_device * dev)4237  static inline void netif_testing_off(struct net_device *dev)
4238  {
4239  	if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
4240  		linkwatch_fire_event(dev);
4241  }
4242  
4243  /**
4244   *	netif_testing - test if device is under test
4245   *	@dev: network device
4246   *
4247   * Check if device is under test
4248   */
netif_testing(const struct net_device * dev)4249  static inline bool netif_testing(const struct net_device *dev)
4250  {
4251  	return test_bit(__LINK_STATE_TESTING, &dev->state);
4252  }
4253  
4254  
4255  /**
4256   *	netif_oper_up - test if device is operational
4257   *	@dev: network device
4258   *
4259   * Check if carrier is operational
4260   */
netif_oper_up(const struct net_device * dev)4261  static inline bool netif_oper_up(const struct net_device *dev)
4262  {
4263  	return (dev->operstate == IF_OPER_UP ||
4264  		dev->operstate == IF_OPER_UNKNOWN /* backward compat */);
4265  }
4266  
4267  /**
4268   *	netif_device_present - is device available or removed
4269   *	@dev: network device
4270   *
4271   * Check if device has not been removed from system.
4272   */
netif_device_present(const struct net_device * dev)4273  static inline bool netif_device_present(const struct net_device *dev)
4274  {
4275  	return test_bit(__LINK_STATE_PRESENT, &dev->state);
4276  }
4277  
4278  void netif_device_detach(struct net_device *dev);
4279  
4280  void netif_device_attach(struct net_device *dev);
4281  
4282  /*
4283   * Network interface message level settings
4284   */
4285  
4286  enum {
4287  	NETIF_MSG_DRV_BIT,
4288  	NETIF_MSG_PROBE_BIT,
4289  	NETIF_MSG_LINK_BIT,
4290  	NETIF_MSG_TIMER_BIT,
4291  	NETIF_MSG_IFDOWN_BIT,
4292  	NETIF_MSG_IFUP_BIT,
4293  	NETIF_MSG_RX_ERR_BIT,
4294  	NETIF_MSG_TX_ERR_BIT,
4295  	NETIF_MSG_TX_QUEUED_BIT,
4296  	NETIF_MSG_INTR_BIT,
4297  	NETIF_MSG_TX_DONE_BIT,
4298  	NETIF_MSG_RX_STATUS_BIT,
4299  	NETIF_MSG_PKTDATA_BIT,
4300  	NETIF_MSG_HW_BIT,
4301  	NETIF_MSG_WOL_BIT,
4302  
4303  	/* When you add a new bit above, update netif_msg_class_names array
4304  	 * in net/ethtool/common.c
4305  	 */
4306  	NETIF_MSG_CLASS_COUNT,
4307  };
4308  /* Both ethtool_ops interface and internal driver implementation use u32 */
4309  static_assert(NETIF_MSG_CLASS_COUNT <= 32);
4310  
4311  #define __NETIF_MSG_BIT(bit)	((u32)1 << (bit))
4312  #define __NETIF_MSG(name)	__NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)
4313  
4314  #define NETIF_MSG_DRV		__NETIF_MSG(DRV)
4315  #define NETIF_MSG_PROBE		__NETIF_MSG(PROBE)
4316  #define NETIF_MSG_LINK		__NETIF_MSG(LINK)
4317  #define NETIF_MSG_TIMER		__NETIF_MSG(TIMER)
4318  #define NETIF_MSG_IFDOWN	__NETIF_MSG(IFDOWN)
4319  #define NETIF_MSG_IFUP		__NETIF_MSG(IFUP)
4320  #define NETIF_MSG_RX_ERR	__NETIF_MSG(RX_ERR)
4321  #define NETIF_MSG_TX_ERR	__NETIF_MSG(TX_ERR)
4322  #define NETIF_MSG_TX_QUEUED	__NETIF_MSG(TX_QUEUED)
4323  #define NETIF_MSG_INTR		__NETIF_MSG(INTR)
4324  #define NETIF_MSG_TX_DONE	__NETIF_MSG(TX_DONE)
4325  #define NETIF_MSG_RX_STATUS	__NETIF_MSG(RX_STATUS)
4326  #define NETIF_MSG_PKTDATA	__NETIF_MSG(PKTDATA)
4327  #define NETIF_MSG_HW		__NETIF_MSG(HW)
4328  #define NETIF_MSG_WOL		__NETIF_MSG(WOL)
4329  
4330  #define netif_msg_drv(p)	((p)->msg_enable & NETIF_MSG_DRV)
4331  #define netif_msg_probe(p)	((p)->msg_enable & NETIF_MSG_PROBE)
4332  #define netif_msg_link(p)	((p)->msg_enable & NETIF_MSG_LINK)
4333  #define netif_msg_timer(p)	((p)->msg_enable & NETIF_MSG_TIMER)
4334  #define netif_msg_ifdown(p)	((p)->msg_enable & NETIF_MSG_IFDOWN)
4335  #define netif_msg_ifup(p)	((p)->msg_enable & NETIF_MSG_IFUP)
4336  #define netif_msg_rx_err(p)	((p)->msg_enable & NETIF_MSG_RX_ERR)
4337  #define netif_msg_tx_err(p)	((p)->msg_enable & NETIF_MSG_TX_ERR)
4338  #define netif_msg_tx_queued(p)	((p)->msg_enable & NETIF_MSG_TX_QUEUED)
4339  #define netif_msg_intr(p)	((p)->msg_enable & NETIF_MSG_INTR)
4340  #define netif_msg_tx_done(p)	((p)->msg_enable & NETIF_MSG_TX_DONE)
4341  #define netif_msg_rx_status(p)	((p)->msg_enable & NETIF_MSG_RX_STATUS)
4342  #define netif_msg_pktdata(p)	((p)->msg_enable & NETIF_MSG_PKTDATA)
4343  #define netif_msg_hw(p)		((p)->msg_enable & NETIF_MSG_HW)
4344  #define netif_msg_wol(p)	((p)->msg_enable & NETIF_MSG_WOL)
4345  
netif_msg_init(int debug_value,int default_msg_enable_bits)4346  static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
4347  {
4348  	/* use default */
4349  	if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
4350  		return default_msg_enable_bits;
4351  	if (debug_value == 0)	/* no output */
4352  		return 0;
4353  	/* set low N bits */
4354  	return (1U << debug_value) - 1;
4355  }
4356  
__netif_tx_lock(struct netdev_queue * txq,int cpu)4357  static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
4358  {
4359  	spin_lock(&txq->_xmit_lock);
4360  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4361  	WRITE_ONCE(txq->xmit_lock_owner, cpu);
4362  }
4363  
__netif_tx_acquire(struct netdev_queue * txq)4364  static inline bool __netif_tx_acquire(struct netdev_queue *txq)
4365  {
4366  	__acquire(&txq->_xmit_lock);
4367  	return true;
4368  }
4369  
__netif_tx_release(struct netdev_queue * txq)4370  static inline void __netif_tx_release(struct netdev_queue *txq)
4371  {
4372  	__release(&txq->_xmit_lock);
4373  }
4374  
__netif_tx_lock_bh(struct netdev_queue * txq)4375  static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
4376  {
4377  	spin_lock_bh(&txq->_xmit_lock);
4378  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4379  	WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4380  }
4381  
__netif_tx_trylock(struct netdev_queue * txq)4382  static inline bool __netif_tx_trylock(struct netdev_queue *txq)
4383  {
4384  	bool ok = spin_trylock(&txq->_xmit_lock);
4385  
4386  	if (likely(ok)) {
4387  		/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4388  		WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4389  	}
4390  	return ok;
4391  }
4392  
__netif_tx_unlock(struct netdev_queue * txq)4393  static inline void __netif_tx_unlock(struct netdev_queue *txq)
4394  {
4395  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4396  	WRITE_ONCE(txq->xmit_lock_owner, -1);
4397  	spin_unlock(&txq->_xmit_lock);
4398  }
4399  
__netif_tx_unlock_bh(struct netdev_queue * txq)4400  static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
4401  {
4402  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4403  	WRITE_ONCE(txq->xmit_lock_owner, -1);
4404  	spin_unlock_bh(&txq->_xmit_lock);
4405  }
4406  
4407  /*
4408   * txq->trans_start can be read locklessly from dev_watchdog()
4409   */
txq_trans_update(struct netdev_queue * txq)4410  static inline void txq_trans_update(struct netdev_queue *txq)
4411  {
4412  	if (txq->xmit_lock_owner != -1)
4413  		WRITE_ONCE(txq->trans_start, jiffies);
4414  }
4415  
txq_trans_cond_update(struct netdev_queue * txq)4416  static inline void txq_trans_cond_update(struct netdev_queue *txq)
4417  {
4418  	unsigned long now = jiffies;
4419  
4420  	if (READ_ONCE(txq->trans_start) != now)
4421  		WRITE_ONCE(txq->trans_start, now);
4422  }
4423  
4424  /* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
netif_trans_update(struct net_device * dev)4425  static inline void netif_trans_update(struct net_device *dev)
4426  {
4427  	struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
4428  
4429  	txq_trans_cond_update(txq);
4430  }
4431  
4432  /**
4433   *	netif_tx_lock - grab network device transmit lock
4434   *	@dev: network device
4435   *
4436   * Get network device transmit lock
4437   */
4438  void netif_tx_lock(struct net_device *dev);
4439  
netif_tx_lock_bh(struct net_device * dev)4440  static inline void netif_tx_lock_bh(struct net_device *dev)
4441  {
4442  	local_bh_disable();
4443  	netif_tx_lock(dev);
4444  }
4445  
4446  void netif_tx_unlock(struct net_device *dev);
4447  
netif_tx_unlock_bh(struct net_device * dev)4448  static inline void netif_tx_unlock_bh(struct net_device *dev)
4449  {
4450  	netif_tx_unlock(dev);
4451  	local_bh_enable();
4452  }
4453  
4454  #define HARD_TX_LOCK(dev, txq, cpu) {			\
4455  	if ((dev->features & NETIF_F_LLTX) == 0) {	\
4456  		__netif_tx_lock(txq, cpu);		\
4457  	} else {					\
4458  		__netif_tx_acquire(txq);		\
4459  	}						\
4460  }
4461  
4462  #define HARD_TX_TRYLOCK(dev, txq)			\
4463  	(((dev->features & NETIF_F_LLTX) == 0) ?	\
4464  		__netif_tx_trylock(txq) :		\
4465  		__netif_tx_acquire(txq))
4466  
4467  #define HARD_TX_UNLOCK(dev, txq) {			\
4468  	if ((dev->features & NETIF_F_LLTX) == 0) {	\
4469  		__netif_tx_unlock(txq);			\
4470  	} else {					\
4471  		__netif_tx_release(txq);		\
4472  	}						\
4473  }
4474  
netif_tx_disable(struct net_device * dev)4475  static inline void netif_tx_disable(struct net_device *dev)
4476  {
4477  	unsigned int i;
4478  	int cpu;
4479  
4480  	local_bh_disable();
4481  	cpu = smp_processor_id();
4482  	spin_lock(&dev->tx_global_lock);
4483  	for (i = 0; i < dev->num_tx_queues; i++) {
4484  		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4485  
4486  		__netif_tx_lock(txq, cpu);
4487  		netif_tx_stop_queue(txq);
4488  		__netif_tx_unlock(txq);
4489  	}
4490  	spin_unlock(&dev->tx_global_lock);
4491  	local_bh_enable();
4492  }
4493  
netif_addr_lock(struct net_device * dev)4494  static inline void netif_addr_lock(struct net_device *dev)
4495  {
4496  	unsigned char nest_level = 0;
4497  
4498  #ifdef CONFIG_LOCKDEP
4499  	nest_level = dev->nested_level;
4500  #endif
4501  	spin_lock_nested(&dev->addr_list_lock, nest_level);
4502  }
4503  
netif_addr_lock_bh(struct net_device * dev)4504  static inline void netif_addr_lock_bh(struct net_device *dev)
4505  {
4506  	unsigned char nest_level = 0;
4507  
4508  #ifdef CONFIG_LOCKDEP
4509  	nest_level = dev->nested_level;
4510  #endif
4511  	local_bh_disable();
4512  	spin_lock_nested(&dev->addr_list_lock, nest_level);
4513  }
4514  
netif_addr_unlock(struct net_device * dev)4515  static inline void netif_addr_unlock(struct net_device *dev)
4516  {
4517  	spin_unlock(&dev->addr_list_lock);
4518  }
4519  
netif_addr_unlock_bh(struct net_device * dev)4520  static inline void netif_addr_unlock_bh(struct net_device *dev)
4521  {
4522  	spin_unlock_bh(&dev->addr_list_lock);
4523  }
4524  
4525  /*
4526   * dev_addrs walker. Should be used only for read access. Call with
4527   * rcu_read_lock held.
4528   */
4529  #define for_each_dev_addr(dev, ha) \
4530  		list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
4531  
4532  /* These functions live elsewhere (drivers/net/net_init.c, but related) */
4533  
4534  void ether_setup(struct net_device *dev);
4535  
4536  /* Support for loadable net-drivers */
4537  struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
4538  				    unsigned char name_assign_type,
4539  				    void (*setup)(struct net_device *),
4540  				    unsigned int txqs, unsigned int rxqs);
4541  #define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
4542  	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
4543  
4544  #define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
4545  	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
4546  			 count)
4547  
4548  int register_netdev(struct net_device *dev);
4549  void unregister_netdev(struct net_device *dev);
4550  
4551  int devm_register_netdev(struct device *dev, struct net_device *ndev);
4552  
4553  /* General hardware address lists handling functions */
4554  int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
4555  		   struct netdev_hw_addr_list *from_list, int addr_len);
4556  void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
4557  		      struct netdev_hw_addr_list *from_list, int addr_len);
4558  int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
4559  		       struct net_device *dev,
4560  		       int (*sync)(struct net_device *, const unsigned char *),
4561  		       int (*unsync)(struct net_device *,
4562  				     const unsigned char *));
4563  int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
4564  			   struct net_device *dev,
4565  			   int (*sync)(struct net_device *,
4566  				       const unsigned char *, int),
4567  			   int (*unsync)(struct net_device *,
4568  					 const unsigned char *, int));
4569  void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
4570  			      struct net_device *dev,
4571  			      int (*unsync)(struct net_device *,
4572  					    const unsigned char *, int));
4573  void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
4574  			  struct net_device *dev,
4575  			  int (*unsync)(struct net_device *,
4576  					const unsigned char *));
4577  void __hw_addr_init(struct netdev_hw_addr_list *list);
4578  
4579  /* Functions used for device addresses handling */
4580  void dev_addr_mod(struct net_device *dev, unsigned int offset,
4581  		  const void *addr, size_t len);
4582  
4583  static inline void
__dev_addr_set(struct net_device * dev,const void * addr,size_t len)4584  __dev_addr_set(struct net_device *dev, const void *addr, size_t len)
4585  {
4586  	dev_addr_mod(dev, 0, addr, len);
4587  }
4588  
dev_addr_set(struct net_device * dev,const u8 * addr)4589  static inline void dev_addr_set(struct net_device *dev, const u8 *addr)
4590  {
4591  	__dev_addr_set(dev, addr, dev->addr_len);
4592  }
4593  
4594  int dev_addr_add(struct net_device *dev, const unsigned char *addr,
4595  		 unsigned char addr_type);
4596  int dev_addr_del(struct net_device *dev, const unsigned char *addr,
4597  		 unsigned char addr_type);
4598  
4599  /* Functions used for unicast addresses handling */
4600  int dev_uc_add(struct net_device *dev, const unsigned char *addr);
4601  int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
4602  int dev_uc_del(struct net_device *dev, const unsigned char *addr);
4603  int dev_uc_sync(struct net_device *to, struct net_device *from);
4604  int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
4605  void dev_uc_unsync(struct net_device *to, struct net_device *from);
4606  void dev_uc_flush(struct net_device *dev);
4607  void dev_uc_init(struct net_device *dev);
4608  
4609  /**
4610   *  __dev_uc_sync - Synchonize device's unicast list
4611   *  @dev:  device to sync
4612   *  @sync: function to call if address should be added
4613   *  @unsync: function to call if address should be removed
4614   *
4615   *  Add newly added addresses to the interface, and release
4616   *  addresses that have been deleted.
4617   */
__dev_uc_sync(struct net_device * dev,int (* sync)(struct net_device *,const unsigned char *),int (* unsync)(struct net_device *,const unsigned char *))4618  static inline int __dev_uc_sync(struct net_device *dev,
4619  				int (*sync)(struct net_device *,
4620  					    const unsigned char *),
4621  				int (*unsync)(struct net_device *,
4622  					      const unsigned char *))
4623  {
4624  	return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
4625  }
4626  
4627  /**
4628   *  __dev_uc_unsync - Remove synchronized addresses from device
4629   *  @dev:  device to sync
4630   *  @unsync: function to call if address should be removed
4631   *
4632   *  Remove all addresses that were added to the device by dev_uc_sync().
4633   */
__dev_uc_unsync(struct net_device * dev,int (* unsync)(struct net_device *,const unsigned char *))4634  static inline void __dev_uc_unsync(struct net_device *dev,
4635  				   int (*unsync)(struct net_device *,
4636  						 const unsigned char *))
4637  {
4638  	__hw_addr_unsync_dev(&dev->uc, dev, unsync);
4639  }
4640  
4641  /* Functions used for multicast addresses handling */
4642  int dev_mc_add(struct net_device *dev, const unsigned char *addr);
4643  int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
4644  int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
4645  int dev_mc_del(struct net_device *dev, const unsigned char *addr);
4646  int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
4647  int dev_mc_sync(struct net_device *to, struct net_device *from);
4648  int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
4649  void dev_mc_unsync(struct net_device *to, struct net_device *from);
4650  void dev_mc_flush(struct net_device *dev);
4651  void dev_mc_init(struct net_device *dev);
4652  
4653  /**
4654   *  __dev_mc_sync - Synchonize device's multicast list
4655   *  @dev:  device to sync
4656   *  @sync: function to call if address should be added
4657   *  @unsync: function to call if address should be removed
4658   *
4659   *  Add newly added addresses to the interface, and release
4660   *  addresses that have been deleted.
4661   */
__dev_mc_sync(struct net_device * dev,int (* sync)(struct net_device *,const unsigned char *),int (* unsync)(struct net_device *,const unsigned char *))4662  static inline int __dev_mc_sync(struct net_device *dev,
4663  				int (*sync)(struct net_device *,
4664  					    const unsigned char *),
4665  				int (*unsync)(struct net_device *,
4666  					      const unsigned char *))
4667  {
4668  	return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
4669  }
4670  
4671  /**
4672   *  __dev_mc_unsync - Remove synchronized addresses from device
4673   *  @dev:  device to sync
4674   *  @unsync: function to call if address should be removed
4675   *
4676   *  Remove all addresses that were added to the device by dev_mc_sync().
4677   */
__dev_mc_unsync(struct net_device * dev,int (* unsync)(struct net_device *,const unsigned char *))4678  static inline void __dev_mc_unsync(struct net_device *dev,
4679  				   int (*unsync)(struct net_device *,
4680  						 const unsigned char *))
4681  {
4682  	__hw_addr_unsync_dev(&dev->mc, dev, unsync);
4683  }
4684  
4685  /* Functions used for secondary unicast and multicast support */
4686  void dev_set_rx_mode(struct net_device *dev);
4687  int dev_set_promiscuity(struct net_device *dev, int inc);
4688  int dev_set_allmulti(struct net_device *dev, int inc);
4689  void netdev_state_change(struct net_device *dev);
4690  void __netdev_notify_peers(struct net_device *dev);
4691  void netdev_notify_peers(struct net_device *dev);
4692  void netdev_features_change(struct net_device *dev);
4693  /* Load a device via the kmod */
4694  void dev_load(struct net *net, const char *name);
4695  struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
4696  					struct rtnl_link_stats64 *storage);
4697  void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
4698  			     const struct net_device_stats *netdev_stats);
4699  void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
4700  			   const struct pcpu_sw_netstats __percpu *netstats);
4701  void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
4702  
4703  extern int		netdev_max_backlog;
4704  extern int		dev_rx_weight;
4705  extern int		dev_tx_weight;
4706  extern int		gro_normal_batch;
4707  
4708  enum {
4709  	NESTED_SYNC_IMM_BIT,
4710  	NESTED_SYNC_TODO_BIT,
4711  };
4712  
4713  #define __NESTED_SYNC_BIT(bit)	((u32)1 << (bit))
4714  #define __NESTED_SYNC(name)	__NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
4715  
4716  #define NESTED_SYNC_IMM		__NESTED_SYNC(IMM)
4717  #define NESTED_SYNC_TODO	__NESTED_SYNC(TODO)
4718  
4719  struct netdev_nested_priv {
4720  	unsigned char flags;
4721  	void *data;
4722  };
4723  
4724  bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
4725  struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4726  						     struct list_head **iter);
4727  
4728  /* iterate through upper list, must be called under RCU read lock */
4729  #define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
4730  	for (iter = &(dev)->adj_list.upper, \
4731  	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
4732  	     updev; \
4733  	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
4734  
4735  int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
4736  				  int (*fn)(struct net_device *upper_dev,
4737  					    struct netdev_nested_priv *priv),
4738  				  struct netdev_nested_priv *priv);
4739  
4740  bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
4741  				  struct net_device *upper_dev);
4742  
4743  bool netdev_has_any_upper_dev(struct net_device *dev);
4744  
4745  void *netdev_lower_get_next_private(struct net_device *dev,
4746  				    struct list_head **iter);
4747  void *netdev_lower_get_next_private_rcu(struct net_device *dev,
4748  					struct list_head **iter);
4749  
4750  #define netdev_for_each_lower_private(dev, priv, iter) \
4751  	for (iter = (dev)->adj_list.lower.next, \
4752  	     priv = netdev_lower_get_next_private(dev, &(iter)); \
4753  	     priv; \
4754  	     priv = netdev_lower_get_next_private(dev, &(iter)))
4755  
4756  #define netdev_for_each_lower_private_rcu(dev, priv, iter) \
4757  	for (iter = &(dev)->adj_list.lower, \
4758  	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
4759  	     priv; \
4760  	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
4761  
4762  void *netdev_lower_get_next(struct net_device *dev,
4763  				struct list_head **iter);
4764  
4765  #define netdev_for_each_lower_dev(dev, ldev, iter) \
4766  	for (iter = (dev)->adj_list.lower.next, \
4767  	     ldev = netdev_lower_get_next(dev, &(iter)); \
4768  	     ldev; \
4769  	     ldev = netdev_lower_get_next(dev, &(iter)))
4770  
4771  struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
4772  					     struct list_head **iter);
4773  int netdev_walk_all_lower_dev(struct net_device *dev,
4774  			      int (*fn)(struct net_device *lower_dev,
4775  					struct netdev_nested_priv *priv),
4776  			      struct netdev_nested_priv *priv);
4777  int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
4778  				  int (*fn)(struct net_device *lower_dev,
4779  					    struct netdev_nested_priv *priv),
4780  				  struct netdev_nested_priv *priv);
4781  
4782  void *netdev_adjacent_get_private(struct list_head *adj_list);
4783  void *netdev_lower_get_first_private_rcu(struct net_device *dev);
4784  struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
4785  struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
4786  int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
4787  			  struct netlink_ext_ack *extack);
4788  int netdev_master_upper_dev_link(struct net_device *dev,
4789  				 struct net_device *upper_dev,
4790  				 void *upper_priv, void *upper_info,
4791  				 struct netlink_ext_ack *extack);
4792  void netdev_upper_dev_unlink(struct net_device *dev,
4793  			     struct net_device *upper_dev);
4794  int netdev_adjacent_change_prepare(struct net_device *old_dev,
4795  				   struct net_device *new_dev,
4796  				   struct net_device *dev,
4797  				   struct netlink_ext_ack *extack);
4798  void netdev_adjacent_change_commit(struct net_device *old_dev,
4799  				   struct net_device *new_dev,
4800  				   struct net_device *dev);
4801  void netdev_adjacent_change_abort(struct net_device *old_dev,
4802  				  struct net_device *new_dev,
4803  				  struct net_device *dev);
4804  void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
4805  void *netdev_lower_dev_get_private(struct net_device *dev,
4806  				   struct net_device *lower_dev);
4807  void netdev_lower_state_changed(struct net_device *lower_dev,
4808  				void *lower_state_info);
4809  
4810  /* RSS keys are 40 or 52 bytes long */
4811  #define NETDEV_RSS_KEY_LEN 52
4812  extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
4813  void netdev_rss_key_fill(void *buffer, size_t len);
4814  
4815  int skb_checksum_help(struct sk_buff *skb);
4816  int skb_crc32c_csum_help(struct sk_buff *skb);
4817  int skb_csum_hwoffload_help(struct sk_buff *skb,
4818  			    const netdev_features_t features);
4819  
4820  struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
4821  				  netdev_features_t features, bool tx_path);
4822  struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb,
4823  				    netdev_features_t features, __be16 type);
4824  struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
4825  				    netdev_features_t features);
4826  
4827  struct netdev_bonding_info {
4828  	ifslave	slave;
4829  	ifbond	master;
4830  };
4831  
4832  struct netdev_notifier_bonding_info {
4833  	struct netdev_notifier_info info; /* must be first */
4834  	struct netdev_bonding_info  bonding_info;
4835  };
4836  
4837  void netdev_bonding_info_change(struct net_device *dev,
4838  				struct netdev_bonding_info *bonding_info);
4839  
4840  #if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
4841  void ethtool_notify(struct net_device *dev, unsigned int cmd, const void *data);
4842  #else
ethtool_notify(struct net_device * dev,unsigned int cmd,const void * data)4843  static inline void ethtool_notify(struct net_device *dev, unsigned int cmd,
4844  				  const void *data)
4845  {
4846  }
4847  #endif
4848  
4849  static inline
skb_gso_segment(struct sk_buff * skb,netdev_features_t features)4850  struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
4851  {
4852  	return __skb_gso_segment(skb, features, true);
4853  }
4854  __be16 skb_network_protocol(struct sk_buff *skb, int *depth);
4855  
can_checksum_protocol(netdev_features_t features,__be16 protocol)4856  static inline bool can_checksum_protocol(netdev_features_t features,
4857  					 __be16 protocol)
4858  {
4859  	if (protocol == htons(ETH_P_FCOE))
4860  		return !!(features & NETIF_F_FCOE_CRC);
4861  
4862  	/* Assume this is an IP checksum (not SCTP CRC) */
4863  
4864  	if (features & NETIF_F_HW_CSUM) {
4865  		/* Can checksum everything */
4866  		return true;
4867  	}
4868  
4869  	switch (protocol) {
4870  	case htons(ETH_P_IP):
4871  		return !!(features & NETIF_F_IP_CSUM);
4872  	case htons(ETH_P_IPV6):
4873  		return !!(features & NETIF_F_IPV6_CSUM);
4874  	default:
4875  		return false;
4876  	}
4877  }
4878  
4879  #ifdef CONFIG_BUG
4880  void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
4881  #else
netdev_rx_csum_fault(struct net_device * dev,struct sk_buff * skb)4882  static inline void netdev_rx_csum_fault(struct net_device *dev,
4883  					struct sk_buff *skb)
4884  {
4885  }
4886  #endif
4887  /* rx skb timestamps */
4888  void net_enable_timestamp(void);
4889  void net_disable_timestamp(void);
4890  
netdev_get_tstamp(struct net_device * dev,const struct skb_shared_hwtstamps * hwtstamps,bool cycles)4891  static inline ktime_t netdev_get_tstamp(struct net_device *dev,
4892  					const struct skb_shared_hwtstamps *hwtstamps,
4893  					bool cycles)
4894  {
4895  	const struct net_device_ops *ops = dev->netdev_ops;
4896  
4897  	if (ops->ndo_get_tstamp)
4898  		return ops->ndo_get_tstamp(dev, hwtstamps, cycles);
4899  
4900  	return hwtstamps->hwtstamp;
4901  }
4902  
__netdev_start_xmit(const struct net_device_ops * ops,struct sk_buff * skb,struct net_device * dev,bool more)4903  static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
4904  					      struct sk_buff *skb, struct net_device *dev,
4905  					      bool more)
4906  {
4907  	__this_cpu_write(softnet_data.xmit.more, more);
4908  	return ops->ndo_start_xmit(skb, dev);
4909  }
4910  
netdev_xmit_more(void)4911  static inline bool netdev_xmit_more(void)
4912  {
4913  	return __this_cpu_read(softnet_data.xmit.more);
4914  }
4915  
netdev_start_xmit(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq,bool more)4916  static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
4917  					    struct netdev_queue *txq, bool more)
4918  {
4919  	const struct net_device_ops *ops = dev->netdev_ops;
4920  	netdev_tx_t rc;
4921  
4922  	rc = __netdev_start_xmit(ops, skb, dev, more);
4923  	if (rc == NETDEV_TX_OK)
4924  		txq_trans_update(txq);
4925  
4926  	return rc;
4927  }
4928  
4929  int netdev_class_create_file_ns(const struct class_attribute *class_attr,
4930  				const void *ns);
4931  void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
4932  				 const void *ns);
4933  
4934  extern const struct kobj_ns_type_operations net_ns_type_operations;
4935  
4936  const char *netdev_drivername(const struct net_device *dev);
4937  
netdev_intersect_features(netdev_features_t f1,netdev_features_t f2)4938  static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
4939  							  netdev_features_t f2)
4940  {
4941  	if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
4942  		if (f1 & NETIF_F_HW_CSUM)
4943  			f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4944  		else
4945  			f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4946  	}
4947  
4948  	return f1 & f2;
4949  }
4950  
netdev_get_wanted_features(struct net_device * dev)4951  static inline netdev_features_t netdev_get_wanted_features(
4952  	struct net_device *dev)
4953  {
4954  	return (dev->features & ~dev->hw_features) | dev->wanted_features;
4955  }
4956  netdev_features_t netdev_increment_features(netdev_features_t all,
4957  	netdev_features_t one, netdev_features_t mask);
4958  
4959  /* Allow TSO being used on stacked device :
4960   * Performing the GSO segmentation before last device
4961   * is a performance improvement.
4962   */
netdev_add_tso_features(netdev_features_t features,netdev_features_t mask)4963  static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
4964  							netdev_features_t mask)
4965  {
4966  	return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
4967  }
4968  
4969  int __netdev_update_features(struct net_device *dev);
4970  void netdev_update_features(struct net_device *dev);
4971  void netdev_change_features(struct net_device *dev);
4972  
4973  void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4974  					struct net_device *dev);
4975  
4976  netdev_features_t passthru_features_check(struct sk_buff *skb,
4977  					  struct net_device *dev,
4978  					  netdev_features_t features);
4979  netdev_features_t netif_skb_features(struct sk_buff *skb);
4980  
net_gso_ok(netdev_features_t features,int gso_type)4981  static inline bool net_gso_ok(netdev_features_t features, int gso_type)
4982  {
4983  	netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
4984  
4985  	/* check flags correspondence */
4986  	BUILD_BUG_ON(SKB_GSO_TCPV4   != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
4987  	BUILD_BUG_ON(SKB_GSO_DODGY   != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
4988  	BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
4989  	BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
4990  	BUILD_BUG_ON(SKB_GSO_TCPV6   != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
4991  	BUILD_BUG_ON(SKB_GSO_FCOE    != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
4992  	BUILD_BUG_ON(SKB_GSO_GRE     != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
4993  	BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
4994  	BUILD_BUG_ON(SKB_GSO_IPXIP4  != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
4995  	BUILD_BUG_ON(SKB_GSO_IPXIP6  != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
4996  	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
4997  	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
4998  	BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
4999  	BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
5000  	BUILD_BUG_ON(SKB_GSO_SCTP    != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
5001  	BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
5002  	BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
5003  	BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
5004  	BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
5005  
5006  	return (features & feature) == feature;
5007  }
5008  
skb_gso_ok(struct sk_buff * skb,netdev_features_t features)5009  static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
5010  {
5011  	return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
5012  	       (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
5013  }
5014  
netif_needs_gso(struct sk_buff * skb,netdev_features_t features)5015  static inline bool netif_needs_gso(struct sk_buff *skb,
5016  				   netdev_features_t features)
5017  {
5018  	return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
5019  		unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
5020  			 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
5021  }
5022  
5023  void netif_set_tso_max_size(struct net_device *dev, unsigned int size);
5024  void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs);
5025  void netif_inherit_tso_max(struct net_device *to,
5026  			   const struct net_device *from);
5027  
skb_gso_error_unwind(struct sk_buff * skb,__be16 protocol,int pulled_hlen,u16 mac_offset,int mac_len)5028  static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
5029  					int pulled_hlen, u16 mac_offset,
5030  					int mac_len)
5031  {
5032  	skb->protocol = protocol;
5033  	skb->encapsulation = 1;
5034  	skb_push(skb, pulled_hlen);
5035  	skb_reset_transport_header(skb);
5036  	skb->mac_header = mac_offset;
5037  	skb->network_header = skb->mac_header + mac_len;
5038  	skb->mac_len = mac_len;
5039  }
5040  
netif_is_macsec(const struct net_device * dev)5041  static inline bool netif_is_macsec(const struct net_device *dev)
5042  {
5043  	return dev->priv_flags & IFF_MACSEC;
5044  }
5045  
netif_is_macvlan(const struct net_device * dev)5046  static inline bool netif_is_macvlan(const struct net_device *dev)
5047  {
5048  	return dev->priv_flags & IFF_MACVLAN;
5049  }
5050  
netif_is_macvlan_port(const struct net_device * dev)5051  static inline bool netif_is_macvlan_port(const struct net_device *dev)
5052  {
5053  	return dev->priv_flags & IFF_MACVLAN_PORT;
5054  }
5055  
netif_is_bond_master(const struct net_device * dev)5056  static inline bool netif_is_bond_master(const struct net_device *dev)
5057  {
5058  	return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
5059  }
5060  
netif_is_bond_slave(const struct net_device * dev)5061  static inline bool netif_is_bond_slave(const struct net_device *dev)
5062  {
5063  	return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
5064  }
5065  
netif_supports_nofcs(struct net_device * dev)5066  static inline bool netif_supports_nofcs(struct net_device *dev)
5067  {
5068  	return dev->priv_flags & IFF_SUPP_NOFCS;
5069  }
5070  
netif_has_l3_rx_handler(const struct net_device * dev)5071  static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
5072  {
5073  	return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
5074  }
5075  
netif_is_l3_master(const struct net_device * dev)5076  static inline bool netif_is_l3_master(const struct net_device *dev)
5077  {
5078  	return dev->priv_flags & IFF_L3MDEV_MASTER;
5079  }
5080  
netif_is_l3_slave(const struct net_device * dev)5081  static inline bool netif_is_l3_slave(const struct net_device *dev)
5082  {
5083  	return dev->priv_flags & IFF_L3MDEV_SLAVE;
5084  }
5085  
dev_sdif(const struct net_device * dev)5086  static inline int dev_sdif(const struct net_device *dev)
5087  {
5088  #ifdef CONFIG_NET_L3_MASTER_DEV
5089  	if (netif_is_l3_slave(dev))
5090  		return dev->ifindex;
5091  #endif
5092  	return 0;
5093  }
5094  
netif_is_bridge_master(const struct net_device * dev)5095  static inline bool netif_is_bridge_master(const struct net_device *dev)
5096  {
5097  	return dev->priv_flags & IFF_EBRIDGE;
5098  }
5099  
netif_is_bridge_port(const struct net_device * dev)5100  static inline bool netif_is_bridge_port(const struct net_device *dev)
5101  {
5102  	return dev->priv_flags & IFF_BRIDGE_PORT;
5103  }
5104  
netif_is_ovs_master(const struct net_device * dev)5105  static inline bool netif_is_ovs_master(const struct net_device *dev)
5106  {
5107  	return dev->priv_flags & IFF_OPENVSWITCH;
5108  }
5109  
netif_is_ovs_port(const struct net_device * dev)5110  static inline bool netif_is_ovs_port(const struct net_device *dev)
5111  {
5112  	return dev->priv_flags & IFF_OVS_DATAPATH;
5113  }
5114  
netif_is_any_bridge_port(const struct net_device * dev)5115  static inline bool netif_is_any_bridge_port(const struct net_device *dev)
5116  {
5117  	return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
5118  }
5119  
netif_is_team_master(const struct net_device * dev)5120  static inline bool netif_is_team_master(const struct net_device *dev)
5121  {
5122  	return dev->priv_flags & IFF_TEAM;
5123  }
5124  
netif_is_team_port(const struct net_device * dev)5125  static inline bool netif_is_team_port(const struct net_device *dev)
5126  {
5127  	return dev->priv_flags & IFF_TEAM_PORT;
5128  }
5129  
netif_is_lag_master(const struct net_device * dev)5130  static inline bool netif_is_lag_master(const struct net_device *dev)
5131  {
5132  	return netif_is_bond_master(dev) || netif_is_team_master(dev);
5133  }
5134  
netif_is_lag_port(const struct net_device * dev)5135  static inline bool netif_is_lag_port(const struct net_device *dev)
5136  {
5137  	return netif_is_bond_slave(dev) || netif_is_team_port(dev);
5138  }
5139  
netif_is_rxfh_configured(const struct net_device * dev)5140  static inline bool netif_is_rxfh_configured(const struct net_device *dev)
5141  {
5142  	return dev->priv_flags & IFF_RXFH_CONFIGURED;
5143  }
5144  
netif_is_failover(const struct net_device * dev)5145  static inline bool netif_is_failover(const struct net_device *dev)
5146  {
5147  	return dev->priv_flags & IFF_FAILOVER;
5148  }
5149  
netif_is_failover_slave(const struct net_device * dev)5150  static inline bool netif_is_failover_slave(const struct net_device *dev)
5151  {
5152  	return dev->priv_flags & IFF_FAILOVER_SLAVE;
5153  }
5154  
5155  /* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
netif_keep_dst(struct net_device * dev)5156  static inline void netif_keep_dst(struct net_device *dev)
5157  {
5158  	dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
5159  }
5160  
5161  /* return true if dev can't cope with mtu frames that need vlan tag insertion */
netif_reduces_vlan_mtu(struct net_device * dev)5162  static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
5163  {
5164  	/* TODO: reserve and use an additional IFF bit, if we get more users */
5165  	return netif_is_macsec(dev);
5166  }
5167  
5168  extern struct pernet_operations __net_initdata loopback_net_ops;
5169  
5170  /* Logging, debugging and troubleshooting/diagnostic helpers. */
5171  
5172  /* netdev_printk helpers, similar to dev_printk */
5173  
netdev_name(const struct net_device * dev)5174  static inline const char *netdev_name(const struct net_device *dev)
5175  {
5176  	if (!dev->name[0] || strchr(dev->name, '%'))
5177  		return "(unnamed net_device)";
5178  	return dev->name;
5179  }
5180  
netdev_unregistering(const struct net_device * dev)5181  static inline bool netdev_unregistering(const struct net_device *dev)
5182  {
5183  	return dev->reg_state == NETREG_UNREGISTERING;
5184  }
5185  
netdev_reg_state(const struct net_device * dev)5186  static inline const char *netdev_reg_state(const struct net_device *dev)
5187  {
5188  	switch (dev->reg_state) {
5189  	case NETREG_UNINITIALIZED: return " (uninitialized)";
5190  	case NETREG_REGISTERED: return "";
5191  	case NETREG_UNREGISTERING: return " (unregistering)";
5192  	case NETREG_UNREGISTERED: return " (unregistered)";
5193  	case NETREG_RELEASED: return " (released)";
5194  	case NETREG_DUMMY: return " (dummy)";
5195  	}
5196  
5197  	WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
5198  	return " (unknown)";
5199  }
5200  
5201  #define MODULE_ALIAS_NETDEV(device) \
5202  	MODULE_ALIAS("netdev-" device)
5203  
5204  /*
5205   * netdev_WARN() acts like dev_printk(), but with the key difference
5206   * of using a WARN/WARN_ON to get the message out, including the
5207   * file/line information and a backtrace.
5208   */
5209  #define netdev_WARN(dev, format, args...)			\
5210  	WARN(1, "netdevice: %s%s: " format, netdev_name(dev),	\
5211  	     netdev_reg_state(dev), ##args)
5212  
5213  #define netdev_WARN_ONCE(dev, format, args...)				\
5214  	WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev),	\
5215  		  netdev_reg_state(dev), ##args)
5216  
5217  /*
5218   *	The list of packet types we will receive (as opposed to discard)
5219   *	and the routines to invoke.
5220   *
5221   *	Why 16. Because with 16 the only overlap we get on a hash of the
5222   *	low nibble of the protocol value is RARP/SNAP/X.25.
5223   *
5224   *		0800	IP
5225   *		0001	802.3
5226   *		0002	AX.25
5227   *		0004	802.2
5228   *		8035	RARP
5229   *		0005	SNAP
5230   *		0805	X.25
5231   *		0806	ARP
5232   *		8137	IPX
5233   *		0009	Localtalk
5234   *		86DD	IPv6
5235   */
5236  #define PTYPE_HASH_SIZE	(16)
5237  #define PTYPE_HASH_MASK	(PTYPE_HASH_SIZE - 1)
5238  
5239  extern struct list_head ptype_all __read_mostly;
5240  extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
5241  
5242  extern struct net_device *blackhole_netdev;
5243  
5244  /* Note: Avoid these macros in fast path, prefer per-cpu or per-queue counters. */
5245  #define DEV_STATS_INC(DEV, FIELD) atomic_long_inc(&(DEV)->stats.__##FIELD)
5246  #define DEV_STATS_ADD(DEV, FIELD, VAL) 	\
5247  		atomic_long_add((VAL), &(DEV)->stats.__##FIELD)
5248  #define DEV_STATS_READ(DEV, FIELD) atomic_long_read(&(DEV)->stats.__##FIELD)
5249  
5250  #endif	/* _LINUX_NETDEVICE_H */
5251