Lines Matching refs:dev
165 struct net_device *dev,
234 static void list_netdevice(struct net_device *dev) in list_netdevice() argument
236 struct net *net = dev_net(dev); in list_netdevice()
241 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); in list_netdevice()
242 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); in list_netdevice()
243 hlist_add_head_rcu(&dev->index_hlist, in list_netdevice()
244 dev_index_hash(net, dev->ifindex)); in list_netdevice()
253 static void unlist_netdevice(struct net_device *dev) in unlist_netdevice() argument
259 list_del_rcu(&dev->dev_list); in unlist_netdevice()
260 hlist_del_rcu(&dev->name_hlist); in unlist_netdevice()
261 hlist_del_rcu(&dev->index_hlist); in unlist_netdevice()
264 dev_base_seq_inc(dev_net(dev)); in unlist_netdevice()
313 return pt->dev ? &pt->dev->ptype_all : &ptype_all; in ptype_head()
315 return pt->dev ? &pt->dev->ptype_specific : in ptype_head()
519 int netdev_boot_setup_check(struct net_device *dev) in netdev_boot_setup_check() argument
526 !strcmp(dev->name, s[i].name)) { in netdev_boot_setup_check()
527 dev->irq = s[i].map.irq; in netdev_boot_setup_check()
528 dev->base_addr = s[i].map.base_addr; in netdev_boot_setup_check()
529 dev->mem_start = s[i].map.mem_start; in netdev_boot_setup_check()
530 dev->mem_end = s[i].map.mem_end; in netdev_boot_setup_check()
613 int dev_get_iflink(const struct net_device *dev) in dev_get_iflink() argument
615 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) in dev_get_iflink()
616 return dev->netdev_ops->ndo_get_iflink(dev); in dev_get_iflink()
618 return dev->ifindex; in dev_get_iflink()
631 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) in dev_fill_metadata_dst() argument
635 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) in dev_fill_metadata_dst()
644 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); in dev_fill_metadata_dst()
662 struct net_device *dev; in __dev_get_by_name() local
665 hlist_for_each_entry(dev, head, name_hlist) in __dev_get_by_name()
666 if (!strncmp(dev->name, name, IFNAMSIZ)) in __dev_get_by_name()
667 return dev; in __dev_get_by_name()
687 struct net_device *dev; in dev_get_by_name_rcu() local
690 hlist_for_each_entry_rcu(dev, head, name_hlist) in dev_get_by_name_rcu()
691 if (!strncmp(dev->name, name, IFNAMSIZ)) in dev_get_by_name_rcu()
692 return dev; in dev_get_by_name_rcu()
712 struct net_device *dev; in dev_get_by_name() local
715 dev = dev_get_by_name_rcu(net, name); in dev_get_by_name()
716 if (dev) in dev_get_by_name()
717 dev_hold(dev); in dev_get_by_name()
719 return dev; in dev_get_by_name()
737 struct net_device *dev; in __dev_get_by_index() local
740 hlist_for_each_entry(dev, head, index_hlist) in __dev_get_by_index()
741 if (dev->ifindex == ifindex) in __dev_get_by_index()
742 return dev; in __dev_get_by_index()
761 struct net_device *dev; in dev_get_by_index_rcu() local
764 hlist_for_each_entry_rcu(dev, head, index_hlist) in dev_get_by_index_rcu()
765 if (dev->ifindex == ifindex) in dev_get_by_index_rcu()
766 return dev; in dev_get_by_index_rcu()
786 struct net_device *dev; in dev_get_by_index() local
789 dev = dev_get_by_index_rcu(net, ifindex); in dev_get_by_index()
790 if (dev) in dev_get_by_index()
791 dev_hold(dev); in dev_get_by_index()
793 return dev; in dev_get_by_index()
818 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
830 struct net_device *dev; in netdev_get_name() local
836 dev = dev_get_by_index_rcu(net, ifindex); in netdev_get_name()
837 if (!dev) { in netdev_get_name()
842 strcpy(name, dev->name); in netdev_get_name()
868 struct net_device *dev; in dev_getbyhwaddr_rcu() local
870 for_each_netdev_rcu(net, dev) in dev_getbyhwaddr_rcu()
871 if (dev->type == type && in dev_getbyhwaddr_rcu()
872 !memcmp(dev->dev_addr, ha, dev->addr_len)) in dev_getbyhwaddr_rcu()
873 return dev; in dev_getbyhwaddr_rcu()
881 struct net_device *dev; in __dev_getfirstbyhwtype() local
884 for_each_netdev(net, dev) in __dev_getfirstbyhwtype()
885 if (dev->type == type) in __dev_getfirstbyhwtype()
886 return dev; in __dev_getfirstbyhwtype()
894 struct net_device *dev, *ret = NULL; in dev_getfirstbyhwtype() local
897 for_each_netdev_rcu(net, dev) in dev_getfirstbyhwtype()
898 if (dev->type == type) { in dev_getfirstbyhwtype()
899 dev_hold(dev); in dev_getfirstbyhwtype()
900 ret = dev; in dev_getfirstbyhwtype()
922 struct net_device *dev, *ret; in __dev_get_by_flags() local
927 for_each_netdev(net, dev) { in __dev_get_by_flags()
928 if (((dev->flags ^ if_flags) & mask) == 0) { in __dev_get_by_flags()
929 ret = dev; in __dev_get_by_flags()
1032 struct net_device *dev, in dev_alloc_name_ns() argument
1041 strlcpy(dev->name, buf, IFNAMSIZ); in dev_alloc_name_ns()
1059 int dev_alloc_name(struct net_device *dev, const char *name) in dev_alloc_name() argument
1061 return dev_alloc_name_ns(dev_net(dev), dev, name); in dev_alloc_name()
1065 int dev_get_valid_name(struct net *net, struct net_device *dev, in dev_get_valid_name() argument
1074 return dev_alloc_name_ns(net, dev, name); in dev_get_valid_name()
1077 else if (dev->name != name) in dev_get_valid_name()
1078 strlcpy(dev->name, name, IFNAMSIZ); in dev_get_valid_name()
1092 int dev_change_name(struct net_device *dev, const char *newname) in dev_change_name() argument
1101 BUG_ON(!dev_net(dev)); in dev_change_name()
1103 net = dev_net(dev); in dev_change_name()
1117 if (dev->flags & IFF_UP && in dev_change_name()
1118 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK))) in dev_change_name()
1123 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { in dev_change_name()
1128 memcpy(oldname, dev->name, IFNAMSIZ); in dev_change_name()
1130 err = dev_get_valid_name(net, dev, newname); in dev_change_name()
1137 netdev_info(dev, "renamed from %s\n", oldname); in dev_change_name()
1139 old_assign_type = dev->name_assign_type; in dev_change_name()
1140 dev->name_assign_type = NET_NAME_RENAMED; in dev_change_name()
1143 ret = device_rename(&dev->dev, dev->name); in dev_change_name()
1145 memcpy(dev->name, oldname, IFNAMSIZ); in dev_change_name()
1146 dev->name_assign_type = old_assign_type; in dev_change_name()
1153 netdev_adjacent_rename_links(dev, oldname); in dev_change_name()
1156 hlist_del_rcu(&dev->name_hlist); in dev_change_name()
1162 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); in dev_change_name()
1165 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); in dev_change_name()
1173 memcpy(dev->name, oldname, IFNAMSIZ); in dev_change_name()
1175 dev->name_assign_type = old_assign_type; in dev_change_name()
1180 dev->name, ret); in dev_change_name()
1195 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) in dev_set_alias() argument
1212 rcu_swap_protected(dev->ifalias, new_alias, in dev_set_alias()
1232 int dev_get_alias(const struct net_device *dev, char *name, size_t len) in dev_get_alias() argument
1238 alias = rcu_dereference(dev->ifalias); in dev_get_alias()
1252 void netdev_features_change(struct net_device *dev) in netdev_features_change() argument
1254 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); in netdev_features_change()
1266 void netdev_state_change(struct net_device *dev) in netdev_state_change() argument
1268 if (dev->flags & IFF_UP) { in netdev_state_change()
1270 .info.dev = dev, in netdev_state_change()
1275 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); in netdev_state_change()
1290 void netdev_notify_peers(struct net_device *dev) in netdev_notify_peers() argument
1293 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); in netdev_notify_peers()
1294 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); in netdev_notify_peers()
1299 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) in __dev_open() argument
1301 const struct net_device_ops *ops = dev->netdev_ops; in __dev_open()
1306 if (!netif_device_present(dev)) in __dev_open()
1313 netpoll_poll_disable(dev); in __dev_open()
1315 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack); in __dev_open()
1320 set_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1323 ret = ops->ndo_validate_addr(dev); in __dev_open()
1326 ret = ops->ndo_open(dev); in __dev_open()
1328 netpoll_poll_enable(dev); in __dev_open()
1331 clear_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1333 dev->flags |= IFF_UP; in __dev_open()
1334 dev_set_rx_mode(dev); in __dev_open()
1335 dev_activate(dev); in __dev_open()
1336 add_device_randomness(dev->dev_addr, dev->addr_len); in __dev_open()
1355 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack) in dev_open() argument
1359 if (dev->flags & IFF_UP) in dev_open()
1362 ret = __dev_open(dev, extack); in dev_open()
1366 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); in dev_open()
1367 call_netdevice_notifiers(NETDEV_UP, dev); in dev_open()
1375 struct net_device *dev; in __dev_close_many() local
1380 list_for_each_entry(dev, head, close_list) { in __dev_close_many()
1382 netpoll_poll_disable(dev); in __dev_close_many()
1384 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); in __dev_close_many()
1386 clear_bit(__LINK_STATE_START, &dev->state); in __dev_close_many()
1399 list_for_each_entry(dev, head, close_list) { in __dev_close_many()
1400 const struct net_device_ops *ops = dev->netdev_ops; in __dev_close_many()
1410 ops->ndo_stop(dev); in __dev_close_many()
1412 dev->flags &= ~IFF_UP; in __dev_close_many()
1413 netpoll_poll_enable(dev); in __dev_close_many()
1417 static void __dev_close(struct net_device *dev) in __dev_close() argument
1421 list_add(&dev->close_list, &single); in __dev_close()
1428 struct net_device *dev, *tmp; in dev_close_many() local
1431 list_for_each_entry_safe(dev, tmp, head, close_list) in dev_close_many()
1432 if (!(dev->flags & IFF_UP)) in dev_close_many()
1433 list_del_init(&dev->close_list); in dev_close_many()
1437 list_for_each_entry_safe(dev, tmp, head, close_list) { in dev_close_many()
1438 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); in dev_close_many()
1439 call_netdevice_notifiers(NETDEV_DOWN, dev); in dev_close_many()
1441 list_del_init(&dev->close_list); in dev_close_many()
1455 void dev_close(struct net_device *dev) in dev_close() argument
1457 if (dev->flags & IFF_UP) { in dev_close()
1460 list_add(&dev->close_list, &single); in dev_close()
1476 void dev_disable_lro(struct net_device *dev) in dev_disable_lro() argument
1481 dev->wanted_features &= ~NETIF_F_LRO; in dev_disable_lro()
1482 netdev_update_features(dev); in dev_disable_lro()
1484 if (unlikely(dev->features & NETIF_F_LRO)) in dev_disable_lro()
1485 netdev_WARN(dev, "failed to disable LRO!\n"); in dev_disable_lro()
1487 netdev_for_each_lower_dev(dev, lower_dev, iter) in dev_disable_lro()
1500 static void dev_disable_gro_hw(struct net_device *dev) in dev_disable_gro_hw() argument
1502 dev->wanted_features &= ~NETIF_F_GRO_HW; in dev_disable_gro_hw()
1503 netdev_update_features(dev); in dev_disable_gro_hw()
1505 if (unlikely(dev->features & NETIF_F_GRO_HW)) in dev_disable_gro_hw()
1506 netdev_WARN(dev, "failed to disable GRO_HW!\n"); in dev_disable_gro_hw()
1532 struct net_device *dev) in call_netdevice_notifier() argument
1535 .dev = dev, in call_netdevice_notifier()
1559 struct net_device *dev; in register_netdevice_notifier() local
1573 for_each_netdev(net, dev) { in register_netdevice_notifier()
1574 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); in register_netdevice_notifier()
1579 if (!(dev->flags & IFF_UP)) in register_netdevice_notifier()
1582 call_netdevice_notifier(nb, NETDEV_UP, dev); in register_netdevice_notifier()
1592 last = dev; in register_netdevice_notifier()
1594 for_each_netdev(net, dev) { in register_netdevice_notifier()
1595 if (dev == last) in register_netdevice_notifier()
1598 if (dev->flags & IFF_UP) { in register_netdevice_notifier()
1600 dev); in register_netdevice_notifier()
1601 call_netdevice_notifier(nb, NETDEV_DOWN, dev); in register_netdevice_notifier()
1603 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); in register_netdevice_notifier()
1629 struct net_device *dev; in unregister_netdevice_notifier() local
1641 for_each_netdev(net, dev) { in unregister_netdevice_notifier()
1642 if (dev->flags & IFF_UP) { in unregister_netdevice_notifier()
1644 dev); in unregister_netdevice_notifier()
1645 call_netdevice_notifier(nb, NETDEV_DOWN, dev); in unregister_netdevice_notifier()
1647 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); in unregister_netdevice_notifier()
1674 struct net_device *dev, in call_netdevice_notifiers_extack() argument
1678 .dev = dev, in call_netdevice_notifiers_extack()
1694 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) in call_netdevice_notifiers() argument
1696 return call_netdevice_notifiers_extack(val, dev, NULL); in call_netdevice_notifiers()
1710 struct net_device *dev, u32 arg) in call_netdevice_notifiers_mtu() argument
1713 .info.dev = dev, in call_netdevice_notifiers_mtu()
1825 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) in is_skb_forwardable() argument
1829 if (!(dev->flags & IFF_UP)) in is_skb_forwardable()
1832 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; in is_skb_forwardable()
1846 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) in __dev_forward_skb() argument
1848 int ret = ____dev_forward_skb(dev, skb); in __dev_forward_skb()
1851 skb->protocol = eth_type_trans(skb, dev); in __dev_forward_skb()
1877 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) in dev_forward_skb() argument
1879 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); in dev_forward_skb()
1890 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in deliver_skb()
1929 bool dev_nit_active(struct net_device *dev) in dev_nit_active() argument
1931 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all); in dev_nit_active()
1940 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) in dev_queue_xmit_nit() argument
1960 deliver_skb(skb2, pt_prev, skb->dev); in dev_queue_xmit_nit()
1982 dev->name); in dev_queue_xmit_nit()
1992 ptype_list = &dev->ptype_all; in dev_queue_xmit_nit()
1998 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); in dev_queue_xmit_nit()
2019 static void netif_setup_tc(struct net_device *dev, unsigned int txq) in netif_setup_tc() argument
2022 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netif_setup_tc()
2027 dev->num_tc = 0; in netif_setup_tc()
2033 int q = netdev_get_prio_tc_map(dev, i); in netif_setup_tc()
2035 tc = &dev->tc_to_txq[q]; in netif_setup_tc()
2039 netdev_set_prio_tc_map(dev, i, 0); in netif_setup_tc()
2044 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) in netdev_txq_to_tc() argument
2046 if (dev->num_tc) { in netdev_txq_to_tc()
2047 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netdev_txq_to_tc()
2101 static bool remove_xps_queue_cpu(struct net_device *dev, in remove_xps_queue_cpu() argument
2105 int num_tc = dev->num_tc ? : 1; in remove_xps_queue_cpu()
2123 static void reset_xps_maps(struct net_device *dev, in reset_xps_maps() argument
2129 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL); in reset_xps_maps()
2131 RCU_INIT_POINTER(dev->xps_cpus_map, NULL); in reset_xps_maps()
2137 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask, in clean_xps_maps() argument
2146 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, in clean_xps_maps()
2149 reset_xps_maps(dev, dev_maps, is_rxqs_map); in clean_xps_maps()
2154 netdev_get_tx_queue(dev, i), in clean_xps_maps()
2160 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, in netif_reset_xps_queues() argument
2174 dev_maps = xmap_dereference(dev->xps_rxqs_map); in netif_reset_xps_queues()
2176 nr_ids = dev->num_rx_queues; in netif_reset_xps_queues()
2177 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, in netif_reset_xps_queues()
2182 dev_maps = xmap_dereference(dev->xps_cpus_map); in netif_reset_xps_queues()
2189 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count, in netif_reset_xps_queues()
2197 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) in netif_reset_xps_queues_gt() argument
2199 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); in netif_reset_xps_queues_gt()
2243 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, in __netif_set_xps_queue() argument
2254 WARN_ON_ONCE(index >= dev->num_tx_queues); in __netif_set_xps_queue()
2256 if (dev->num_tc) { in __netif_set_xps_queue()
2258 num_tc = dev->num_tc; in __netif_set_xps_queue()
2263 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; in __netif_set_xps_queue()
2265 tc = netdev_txq_to_tc(dev, index); in __netif_set_xps_queue()
2272 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); in __netif_set_xps_queue()
2273 dev_maps = xmap_dereference(dev->xps_rxqs_map); in __netif_set_xps_queue()
2274 nr_ids = dev->num_rx_queues; in __netif_set_xps_queue()
2281 dev_maps = xmap_dereference(dev->xps_cpus_map); in __netif_set_xps_queue()
2367 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps); in __netif_set_xps_queue()
2369 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps); in __netif_set_xps_queue()
2394 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), in __netif_set_xps_queue()
2416 reset_xps_maps(dev, dev_maps, is_rxqs_map); in __netif_set_xps_queue()
2443 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, in netif_set_xps_queue() argument
2449 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false); in netif_set_xps_queue()
2457 static void netdev_unbind_all_sb_channels(struct net_device *dev) in netdev_unbind_all_sb_channels() argument
2459 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_all_sb_channels()
2462 while (txq-- != &dev->_tx[0]) { in netdev_unbind_all_sb_channels()
2464 netdev_unbind_sb_channel(dev, txq->sb_dev); in netdev_unbind_all_sb_channels()
2468 void netdev_reset_tc(struct net_device *dev) in netdev_reset_tc() argument
2471 netif_reset_xps_queues_gt(dev, 0); in netdev_reset_tc()
2473 netdev_unbind_all_sb_channels(dev); in netdev_reset_tc()
2476 dev->num_tc = 0; in netdev_reset_tc()
2477 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); in netdev_reset_tc()
2478 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); in netdev_reset_tc()
2482 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) in netdev_set_tc_queue() argument
2484 if (tc >= dev->num_tc) in netdev_set_tc_queue()
2488 netif_reset_xps_queues(dev, offset, count); in netdev_set_tc_queue()
2490 dev->tc_to_txq[tc].count = count; in netdev_set_tc_queue()
2491 dev->tc_to_txq[tc].offset = offset; in netdev_set_tc_queue()
2496 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) in netdev_set_num_tc() argument
2502 netif_reset_xps_queues_gt(dev, 0); in netdev_set_num_tc()
2504 netdev_unbind_all_sb_channels(dev); in netdev_set_num_tc()
2506 dev->num_tc = num_tc; in netdev_set_num_tc()
2511 void netdev_unbind_sb_channel(struct net_device *dev, in netdev_unbind_sb_channel() argument
2514 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_sb_channel()
2522 while (txq-- != &dev->_tx[0]) { in netdev_unbind_sb_channel()
2529 int netdev_bind_sb_channel_queue(struct net_device *dev, in netdev_bind_sb_channel_queue() argument
2534 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) in netdev_bind_sb_channel_queue()
2538 if ((offset + count) > dev->real_num_tx_queues) in netdev_bind_sb_channel_queue()
2549 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; in netdev_bind_sb_channel_queue()
2555 int netdev_set_sb_channel(struct net_device *dev, u16 channel) in netdev_set_sb_channel() argument
2558 if (netif_is_multiqueue(dev)) in netdev_set_sb_channel()
2569 dev->num_tc = -channel; in netdev_set_sb_channel()
2579 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) in netif_set_real_num_tx_queues() argument
2584 disabling = txq < dev->real_num_tx_queues; in netif_set_real_num_tx_queues()
2586 if (txq < 1 || txq > dev->num_tx_queues) in netif_set_real_num_tx_queues()
2589 if (dev->reg_state == NETREG_REGISTERED || in netif_set_real_num_tx_queues()
2590 dev->reg_state == NETREG_UNREGISTERING) { in netif_set_real_num_tx_queues()
2593 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, in netif_set_real_num_tx_queues()
2598 if (dev->num_tc) in netif_set_real_num_tx_queues()
2599 netif_setup_tc(dev, txq); in netif_set_real_num_tx_queues()
2601 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
2605 qdisc_reset_all_tx_gt(dev, txq); in netif_set_real_num_tx_queues()
2607 netif_reset_xps_queues_gt(dev, txq); in netif_set_real_num_tx_queues()
2611 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
2629 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) in netif_set_real_num_rx_queues() argument
2633 if (rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_rx_queues()
2636 if (dev->reg_state == NETREG_REGISTERED) { in netif_set_real_num_rx_queues()
2639 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, in netif_set_real_num_rx_queues()
2645 dev->real_num_rx_queues = rxq; in netif_set_real_num_rx_queues()
2759 void netif_device_detach(struct net_device *dev) in netif_device_detach() argument
2761 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_detach()
2762 netif_running(dev)) { in netif_device_detach()
2763 netif_tx_stop_all_queues(dev); in netif_device_detach()
2774 void netif_device_attach(struct net_device *dev) in netif_device_attach() argument
2776 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_attach()
2777 netif_running(dev)) { in netif_device_attach()
2778 netif_tx_wake_all_queues(dev); in netif_device_attach()
2779 __netdev_watchdog_up(dev); in netif_device_attach()
2788 static u16 skb_tx_hash(const struct net_device *dev, in skb_tx_hash() argument
2794 u16 qcount = dev->real_num_tx_queues; in skb_tx_hash()
2796 if (dev->num_tc) { in skb_tx_hash()
2797 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); in skb_tx_hash()
2805 qcount = dev->real_num_tx_queues; in skb_tx_hash()
2824 struct net_device *dev = skb->dev; in skb_warn_bad_offload() local
2830 if (dev) { in skb_warn_bad_offload()
2831 if (dev->dev.parent) in skb_warn_bad_offload()
2832 name = dev_driver_string(dev->dev.parent); in skb_warn_bad_offload()
2834 name = netdev_name(dev); in skb_warn_bad_offload()
2838 name, dev ? &dev->features : &null_features, in skb_warn_bad_offload()
3027 struct net_device *dev = skb->dev; in __skb_gso_segment() local
3029 partial_features |= dev->features & dev->gso_partial_features; in __skb_gso_segment()
3054 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) in netdev_rx_csum_fault() argument
3057 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); in netdev_rx_csum_fault()
3066 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) in illegal_highdma() argument
3071 if (!(dev->features & NETIF_F_HIGHDMA)) { in illegal_highdma()
3092 features &= skb->dev->mpls_features; in net_mpls_features()
3118 if (illegal_highdma(skb->dev, skb)) in harmonize_features()
3125 struct net_device *dev, in passthru_features_check() argument
3133 struct net_device *dev, in dflt_features_check() argument
3140 struct net_device *dev, in gso_features_check() argument
3145 if (gso_segs > dev->gso_max_segs) in gso_features_check()
3148 if (unlikely(skb->len >= READ_ONCE(dev->gso_max_size))) in gso_features_check()
3163 features &= ~dev->gso_partial_features; in gso_features_check()
3181 struct net_device *dev = skb->dev; in netif_skb_features() local
3182 netdev_features_t features = dev->features; in netif_skb_features()
3185 features = gso_features_check(skb, dev, features); in netif_skb_features()
3192 features &= dev->hw_enc_features; in netif_skb_features()
3196 dev->vlan_features | in netif_skb_features()
3200 if (dev->netdev_ops->ndo_features_check) in netif_skb_features()
3201 features &= dev->netdev_ops->ndo_features_check(skb, dev, in netif_skb_features()
3204 features &= dflt_features_check(skb, dev, features); in netif_skb_features()
3210 static int xmit_one(struct sk_buff *skb, struct net_device *dev, in xmit_one() argument
3216 if (dev_nit_active(dev)) in xmit_one()
3217 dev_queue_xmit_nit(skb, dev); in xmit_one()
3220 trace_net_dev_start_xmit(skb, dev); in xmit_one()
3221 rc = netdev_start_xmit(skb, dev, txq, more); in xmit_one()
3222 trace_net_dev_xmit(skb, rc, dev, len); in xmit_one()
3227 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, in dev_hard_start_xmit() argument
3237 rc = xmit_one(skb, dev, txq, next != NULL); in dev_hard_start_xmit()
3275 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) in validate_xmit_skb() argument
3284 skb = sk_validate_xmit_skb(skb, dev); in validate_xmit_skb()
3326 atomic_long_inc(&dev->tx_dropped); in validate_xmit_skb()
3330 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again) in validate_xmit_skb_list() argument
3341 skb = validate_xmit_skb(skb, dev, again); in validate_xmit_skb_list()
3400 struct net_device *dev, in __dev_xmit_skb() argument
3444 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { in __dev_xmit_skb()
3482 map = rcu_dereference_bh(skb->dev->priomap); in skb_update_prio()
3520 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) in sch_handle_egress() argument
3522 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress); in sch_handle_egress()
3561 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb, in __get_xps_queue_idx() argument
3567 if (dev->num_tc) { in __get_xps_queue_idx()
3568 tci *= dev->num_tc; in __get_xps_queue_idx()
3569 tci += netdev_get_prio_tc_map(dev, skb->priority); in __get_xps_queue_idx()
3579 if (unlikely(queue_index >= dev->real_num_tx_queues)) in __get_xps_queue_idx()
3586 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev, in get_xps_queue() argument
3605 if (tci >= 0 && tci < dev->num_rx_queues) in get_xps_queue()
3606 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, in get_xps_queue()
3616 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, in get_xps_queue()
3628 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb, in dev_pick_tx_zero() argument
3635 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb, in dev_pick_tx_cpu_id() argument
3638 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues; in dev_pick_tx_cpu_id()
3642 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb, in netdev_pick_tx() argument
3648 sb_dev = sb_dev ? : dev; in netdev_pick_tx()
3651 queue_index >= dev->real_num_tx_queues) { in netdev_pick_tx()
3652 int new_index = get_xps_queue(dev, sb_dev, skb); in netdev_pick_tx()
3655 new_index = skb_tx_hash(dev, sb_dev, skb); in netdev_pick_tx()
3669 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev, in netdev_core_pick_tx() argument
3682 if (dev->real_num_tx_queues != 1) { in netdev_core_pick_tx()
3683 const struct net_device_ops *ops = dev->netdev_ops; in netdev_core_pick_tx()
3686 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); in netdev_core_pick_tx()
3688 queue_index = netdev_pick_tx(dev, skb, sb_dev); in netdev_core_pick_tx()
3690 queue_index = netdev_cap_txqueue(dev, queue_index); in netdev_core_pick_tx()
3694 return netdev_get_tx_queue(dev, queue_index); in netdev_core_pick_tx()
3725 struct net_device *dev = skb->dev; in __dev_queue_xmit() local
3749 skb = sch_handle_egress(skb, &rc, dev); in __dev_queue_xmit()
3758 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) in __dev_queue_xmit()
3763 txq = netdev_core_pick_tx(dev, skb, sb_dev); in __dev_queue_xmit()
3768 rc = __dev_xmit_skb(skb, q, dev, txq); in __dev_queue_xmit()
3784 if (dev->flags & IFF_UP) { in __dev_queue_xmit()
3794 skb = validate_xmit_skb(skb, dev, &again); in __dev_queue_xmit()
3798 HARD_TX_LOCK(dev, txq, cpu); in __dev_queue_xmit()
3802 skb = dev_hard_start_xmit(skb, dev, txq, &rc); in __dev_queue_xmit()
3805 HARD_TX_UNLOCK(dev, txq); in __dev_queue_xmit()
3809 HARD_TX_UNLOCK(dev, txq); in __dev_queue_xmit()
3811 dev->name); in __dev_queue_xmit()
3818 dev->name); in __dev_queue_xmit()
3825 atomic_long_inc(&dev->tx_dropped); in __dev_queue_xmit()
3847 struct net_device *dev = skb->dev; in dev_direct_xmit() local
3853 if (unlikely(!netif_running(dev) || in dev_direct_xmit()
3854 !netif_carrier_ok(dev))) in dev_direct_xmit()
3857 skb = validate_xmit_skb_list(skb, dev, &again); in dev_direct_xmit()
3862 txq = skb_get_tx_queue(dev, skb); in dev_direct_xmit()
3867 HARD_TX_LOCK(dev, txq, smp_processor_id()); in dev_direct_xmit()
3869 ret = netdev_start_xmit(skb, dev, txq, false); in dev_direct_xmit()
3870 HARD_TX_UNLOCK(dev, txq); in dev_direct_xmit()
3880 atomic_long_inc(&dev->tx_dropped); in dev_direct_xmit()
3927 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, in set_rps_cpu() argument
3940 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || in set_rps_cpu()
3941 !(dev->features & NETIF_F_NTUPLE)) in set_rps_cpu()
3943 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); in set_rps_cpu()
3947 rxqueue = dev->_rx + rxq_index; in set_rps_cpu()
3952 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, in set_rps_cpu()
3976 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, in get_rps_cpu() argument
3980 struct netdev_rx_queue *rxqueue = dev->_rx; in get_rps_cpu()
3990 if (unlikely(index >= dev->real_num_rx_queues)) { in get_rps_cpu()
3991 WARN_ONCE(dev->real_num_rx_queues > 1, in get_rps_cpu()
3994 dev->name, index, dev->real_num_rx_queues); in get_rps_cpu()
4049 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); in get_rps_cpu()
4086 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, in rps_may_expire_flow() argument
4089 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; in rps_may_expire_flow()
4201 if (!netif_running(skb->dev)) in enqueue_to_backlog()
4230 atomic_long_inc(&skb->dev->rx_dropped); in enqueue_to_backlog()
4237 struct net_device *dev = skb->dev; in netif_get_rxqueue() local
4240 rxqueue = dev->_rx; in netif_get_rxqueue()
4245 if (unlikely(index >= dev->real_num_rx_queues)) { in netif_get_rxqueue()
4246 WARN_ONCE(dev->real_num_rx_queues > 1, in netif_get_rxqueue()
4249 dev->name, index, dev->real_num_rx_queues); in netif_get_rxqueue()
4344 skb->protocol = eth_type_trans(skb, skb->dev); in netif_receive_generic_xdp()
4361 trace_xdp_exception(skb->dev, xdp_prog, act); in netif_receive_generic_xdp()
4377 struct net_device *dev = skb->dev; in generic_xdp_tx() local
4382 txq = netdev_core_pick_tx(dev, skb, NULL); in generic_xdp_tx()
4384 HARD_TX_LOCK(dev, txq, cpu); in generic_xdp_tx()
4386 rc = netdev_start_xmit(skb, dev, txq, 0); in generic_xdp_tx()
4390 HARD_TX_UNLOCK(dev, txq); in generic_xdp_tx()
4392 trace_xdp_exception(dev, xdp_prog, XDP_TX); in generic_xdp_tx()
4411 err = xdp_do_generic_redirect(skb->dev, skb, in do_xdp_generic()
4446 cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_rx_internal()
4595 int (*br_fdb_test_addr_hook)(struct net_device *dev,
4605 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress); in sch_handle_ingress()
4665 bool netdev_is_rx_handler_busy(struct net_device *dev) in netdev_is_rx_handler_busy() argument
4668 return dev && rtnl_dereference(dev->rx_handler); in netdev_is_rx_handler_busy()
4686 int netdev_rx_handler_register(struct net_device *dev, in netdev_rx_handler_register() argument
4690 if (netdev_is_rx_handler_busy(dev)) in netdev_rx_handler_register()
4693 if (dev->priv_flags & IFF_NO_RX_HANDLER) in netdev_rx_handler_register()
4697 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); in netdev_rx_handler_register()
4698 rcu_assign_pointer(dev->rx_handler, rx_handler); in netdev_rx_handler_register()
4712 void netdev_rx_handler_unregister(struct net_device *dev) in netdev_rx_handler_unregister() argument
4716 RCU_INIT_POINTER(dev->rx_handler, NULL); in netdev_rx_handler_unregister()
4722 RCU_INIT_POINTER(dev->rx_handler_data, NULL); in netdev_rx_handler_unregister()
4780 orig_dev = skb->dev; in __netif_receive_skb_core()
4790 skb->skb_iif = skb->dev->ifindex; in __netif_receive_skb_core()
4798 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb); in __netif_receive_skb_core()
4827 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { in __netif_receive_skb_core()
4860 rx_handler = rcu_dereference(skb->dev->rx_handler); in __netif_receive_skb_core()
4931 if (unlikely(skb->dev != orig_dev)) { in __netif_receive_skb_core()
4933 &skb->dev->ptype_specific); in __netif_receive_skb_core()
4943 atomic_long_inc(&skb->dev->rx_dropped); in __netif_receive_skb_core()
4945 atomic_long_inc(&skb->dev->rx_nohandler); in __netif_receive_skb_core()
4966 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_one_core()
4973 skb->dev, pt_prev, orig_dev); in __netif_receive_skb_one_core()
5020 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in __netif_receive_skb_list_ptype()
5044 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_list_core()
5121 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) in generic_xdp_install() argument
5123 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); in generic_xdp_install()
5129 rcu_assign_pointer(dev->xdp_prog, new); in generic_xdp_install()
5137 dev_disable_lro(dev); in generic_xdp_install()
5138 dev_disable_gro_hw(dev); in generic_xdp_install()
5167 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_internal()
5200 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_list_internal()
5281 if (skb->dev->reg_state == NETREG_UNREGISTERING) { in flush_backlog()
5291 if (skb->dev->reg_state == NETREG_UNREGISTERING) { in flush_backlog()
5414 unsigned int maclen = skb->dev->hard_header_len; in gro_list_prepare()
5430 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; in gro_list_prepare()
5525 if (netif_elide_gro(skb->dev)) in dev_gro_receive()
5713 skb->dev = napi->dev; in napi_reuse_skb()
5750 skb->protocol = eth_type_trans(skb, skb->dev); in napi_frags_finish()
5793 __func__, napi->dev->name); in napi_frags_skb()
5848 netdev_rx_csum_fault(skb->dev, skb); in __skb_gro_checksum_complete()
6040 timeout = n->dev->gro_flush_timeout; in napi_complete_done()
6184 __NET_ADD_STATS(dev_net(napi->dev), in napi_busy_loop()
6279 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, in netif_napi_add() argument
6291 netdev_err_once(dev, "%s() called with weight %d\n", __func__, in netif_napi_add()
6294 napi->dev = dev; in netif_napi_add()
6300 list_add_rcu(&napi->dev_list, &dev->napi_list); in netif_napi_add()
6400 n->dev ? n->dev->name : "backlog"); in napi_poll()
6462 struct net_device *dev; member
6486 if (adj->dev == adj_dev) in __netdev_find_adj()
6494 struct net_device *dev = data; in ____netdev_has_upper_dev() local
6496 return upper_dev == dev; in ____netdev_has_upper_dev()
6508 bool netdev_has_upper_dev(struct net_device *dev, in netdev_has_upper_dev() argument
6513 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, in netdev_has_upper_dev()
6528 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, in netdev_has_upper_dev_all_rcu() argument
6531 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, in netdev_has_upper_dev_all_rcu()
6543 bool netdev_has_any_upper_dev(struct net_device *dev) in netdev_has_any_upper_dev() argument
6547 return !list_empty(&dev->adj_list.upper); in netdev_has_any_upper_dev()
6558 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) in netdev_master_upper_dev_get() argument
6564 if (list_empty(&dev->adj_list.upper)) in netdev_master_upper_dev_get()
6567 upper = list_first_entry(&dev->adj_list.upper, in netdev_master_upper_dev_get()
6570 return upper->dev; in netdev_master_upper_dev_get()
6575 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev) in __netdev_master_upper_dev_get() argument
6581 if (list_empty(&dev->adj_list.upper)) in __netdev_master_upper_dev_get()
6584 upper = list_first_entry(&dev->adj_list.upper, in __netdev_master_upper_dev_get()
6587 return upper->dev; in __netdev_master_upper_dev_get()
6598 static bool netdev_has_any_lower_dev(struct net_device *dev) in netdev_has_any_lower_dev() argument
6602 return !list_empty(&dev->adj_list.lower); in netdev_has_any_lower_dev()
6623 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, in netdev_upper_get_next_dev_rcu() argument
6632 if (&upper->list == &dev->adj_list.upper) in netdev_upper_get_next_dev_rcu()
6637 return upper->dev; in netdev_upper_get_next_dev_rcu()
6641 static struct net_device *__netdev_next_upper_dev(struct net_device *dev, in __netdev_next_upper_dev() argument
6649 if (&upper->list == &dev->adj_list.upper) in __netdev_next_upper_dev()
6655 return upper->dev; in __netdev_next_upper_dev()
6658 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, in netdev_next_upper_dev_rcu() argument
6667 if (&upper->list == &dev->adj_list.upper) in netdev_next_upper_dev_rcu()
6672 return upper->dev; in netdev_next_upper_dev_rcu()
6675 static int __netdev_walk_all_upper_dev(struct net_device *dev, in __netdev_walk_all_upper_dev() argument
6676 int (*fn)(struct net_device *dev, in __netdev_walk_all_upper_dev() argument
6685 now = dev; in __netdev_walk_all_upper_dev()
6686 iter = &dev->adj_list.upper; in __netdev_walk_all_upper_dev()
6689 if (now != dev) { in __netdev_walk_all_upper_dev()
6724 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, in netdev_walk_all_upper_dev_rcu() argument
6725 int (*fn)(struct net_device *dev, in netdev_walk_all_upper_dev_rcu() argument
6733 now = dev; in netdev_walk_all_upper_dev_rcu()
6734 iter = &dev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
6737 if (now != dev) { in netdev_walk_all_upper_dev_rcu()
6771 static bool __netdev_has_upper_dev(struct net_device *dev, in __netdev_has_upper_dev() argument
6776 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev, in __netdev_has_upper_dev()
6791 void *netdev_lower_get_next_private(struct net_device *dev, in netdev_lower_get_next_private() argument
6798 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private()
6817 void *netdev_lower_get_next_private_rcu(struct net_device *dev, in netdev_lower_get_next_private_rcu() argument
6826 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private_rcu()
6846 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) in netdev_lower_get_next() argument
6852 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next()
6857 return lower->dev; in netdev_lower_get_next()
6861 static struct net_device *netdev_next_lower_dev(struct net_device *dev, in netdev_next_lower_dev() argument
6868 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev()
6873 return lower->dev; in netdev_next_lower_dev()
6876 static struct net_device *__netdev_next_lower_dev(struct net_device *dev, in __netdev_next_lower_dev() argument
6884 if (&lower->list == &dev->adj_list.lower) in __netdev_next_lower_dev()
6890 return lower->dev; in __netdev_next_lower_dev()
6893 int netdev_walk_all_lower_dev(struct net_device *dev, in netdev_walk_all_lower_dev() argument
6894 int (*fn)(struct net_device *dev, in netdev_walk_all_lower_dev() argument
6902 now = dev; in netdev_walk_all_lower_dev()
6903 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev()
6906 if (now != dev) { in netdev_walk_all_lower_dev()
6940 static int __netdev_walk_all_lower_dev(struct net_device *dev, in __netdev_walk_all_lower_dev() argument
6941 int (*fn)(struct net_device *dev, in __netdev_walk_all_lower_dev() argument
6950 now = dev; in __netdev_walk_all_lower_dev()
6951 iter = &dev->adj_list.lower; in __netdev_walk_all_lower_dev()
6954 if (now != dev) { in __netdev_walk_all_lower_dev()
6989 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, in netdev_next_lower_dev_rcu() argument
6995 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev_rcu()
7000 return lower->dev; in netdev_next_lower_dev_rcu()
7004 static u8 __netdev_upper_depth(struct net_device *dev) in __netdev_upper_depth() argument
7011 for (iter = &dev->adj_list.upper, in __netdev_upper_depth()
7012 udev = __netdev_next_upper_dev(dev, &iter, &ignore); in __netdev_upper_depth()
7014 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) { in __netdev_upper_depth()
7024 static u8 __netdev_lower_depth(struct net_device *dev) in __netdev_lower_depth() argument
7031 for (iter = &dev->adj_list.lower, in __netdev_lower_depth()
7032 ldev = __netdev_next_lower_dev(dev, &iter, &ignore); in __netdev_lower_depth()
7034 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) { in __netdev_lower_depth()
7044 static int __netdev_update_upper_level(struct net_device *dev, void *data) in __netdev_update_upper_level() argument
7046 dev->upper_level = __netdev_upper_depth(dev) + 1; in __netdev_update_upper_level()
7050 static int __netdev_update_lower_level(struct net_device *dev, void *data) in __netdev_update_lower_level() argument
7052 dev->lower_level = __netdev_lower_depth(dev) + 1; in __netdev_update_lower_level()
7056 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, in netdev_walk_all_lower_dev_rcu() argument
7057 int (*fn)(struct net_device *dev, in netdev_walk_all_lower_dev_rcu() argument
7065 now = dev; in netdev_walk_all_lower_dev_rcu()
7066 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
7069 if (now != dev) { in netdev_walk_all_lower_dev_rcu()
7112 void *netdev_lower_get_first_private_rcu(struct net_device *dev) in netdev_lower_get_first_private_rcu() argument
7116 lower = list_first_or_null_rcu(&dev->adj_list.lower, in netdev_lower_get_first_private_rcu()
7131 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) in netdev_master_upper_dev_get_rcu() argument
7135 upper = list_first_or_null_rcu(&dev->adj_list.upper, in netdev_master_upper_dev_get_rcu()
7138 return upper->dev; in netdev_master_upper_dev_get_rcu()
7143 static int netdev_adjacent_sysfs_add(struct net_device *dev, in netdev_adjacent_sysfs_add() argument
7149 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_add()
7151 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), in netdev_adjacent_sysfs_add()
7154 static void netdev_adjacent_sysfs_del(struct net_device *dev, in netdev_adjacent_sysfs_del() argument
7160 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_del()
7162 sysfs_remove_link(&(dev->dev.kobj), linkname); in netdev_adjacent_sysfs_del()
7165 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, in netdev_adjacent_is_neigh_list() argument
7169 return (dev_list == &dev->adj_list.upper || in netdev_adjacent_is_neigh_list()
7170 dev_list == &dev->adj_list.lower) && in netdev_adjacent_is_neigh_list()
7171 net_eq(dev_net(dev), dev_net(adj_dev)); in netdev_adjacent_is_neigh_list()
7174 static int __netdev_adjacent_dev_insert(struct net_device *dev, in __netdev_adjacent_dev_insert() argument
7187 dev->name, adj_dev->name, adj->ref_nr); in __netdev_adjacent_dev_insert()
7196 adj->dev = adj_dev; in __netdev_adjacent_dev_insert()
7204 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); in __netdev_adjacent_dev_insert()
7206 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { in __netdev_adjacent_dev_insert()
7207 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); in __netdev_adjacent_dev_insert()
7214 ret = sysfs_create_link(&(dev->dev.kobj), in __netdev_adjacent_dev_insert()
7215 &(adj_dev->dev.kobj), "master"); in __netdev_adjacent_dev_insert()
7227 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) in __netdev_adjacent_dev_insert()
7228 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_insert()
7236 static void __netdev_adjacent_dev_remove(struct net_device *dev, in __netdev_adjacent_dev_remove() argument
7244 dev->name, adj_dev->name, ref_nr); in __netdev_adjacent_dev_remove()
7250 dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
7257 dev->name, adj_dev->name, ref_nr, in __netdev_adjacent_dev_remove()
7264 sysfs_remove_link(&(dev->dev.kobj), "master"); in __netdev_adjacent_dev_remove()
7266 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) in __netdev_adjacent_dev_remove()
7267 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_remove()
7271 adj_dev->name, dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
7276 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, in __netdev_adjacent_dev_link_lists() argument
7284 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, in __netdev_adjacent_dev_link_lists()
7289 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, in __netdev_adjacent_dev_link_lists()
7292 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); in __netdev_adjacent_dev_link_lists()
7299 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, in __netdev_adjacent_dev_unlink_lists() argument
7305 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); in __netdev_adjacent_dev_unlink_lists()
7306 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); in __netdev_adjacent_dev_unlink_lists()
7309 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, in __netdev_adjacent_dev_link_neighbour() argument
7313 return __netdev_adjacent_dev_link_lists(dev, upper_dev, in __netdev_adjacent_dev_link_neighbour()
7314 &dev->adj_list.upper, in __netdev_adjacent_dev_link_neighbour()
7319 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, in __netdev_adjacent_dev_unlink_neighbour() argument
7322 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, in __netdev_adjacent_dev_unlink_neighbour()
7323 &dev->adj_list.upper, in __netdev_adjacent_dev_unlink_neighbour()
7327 static int __netdev_upper_dev_link(struct net_device *dev, in __netdev_upper_dev_link() argument
7334 .dev = dev, in __netdev_upper_dev_link()
7347 if (dev == upper_dev) in __netdev_upper_dev_link()
7351 if (__netdev_has_upper_dev(upper_dev, dev)) in __netdev_upper_dev_link()
7354 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) in __netdev_upper_dev_link()
7358 if (__netdev_has_upper_dev(dev, upper_dev)) in __netdev_upper_dev_link()
7361 master_dev = __netdev_master_upper_dev_get(dev); in __netdev_upper_dev_link()
7372 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, in __netdev_upper_dev_link()
7383 __netdev_update_upper_level(dev, NULL); in __netdev_upper_dev_link()
7384 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); in __netdev_upper_dev_link()
7393 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); in __netdev_upper_dev_link()
7409 int netdev_upper_dev_link(struct net_device *dev, in netdev_upper_dev_link() argument
7413 return __netdev_upper_dev_link(dev, upper_dev, false, in netdev_upper_dev_link()
7432 int netdev_master_upper_dev_link(struct net_device *dev, in netdev_master_upper_dev_link() argument
7437 return __netdev_upper_dev_link(dev, upper_dev, true, in netdev_master_upper_dev_link()
7450 void netdev_upper_dev_unlink(struct net_device *dev, in netdev_upper_dev_unlink() argument
7455 .dev = dev, in netdev_upper_dev_unlink()
7463 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; in netdev_upper_dev_unlink()
7468 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); in netdev_upper_dev_unlink()
7473 __netdev_update_upper_level(dev, NULL); in netdev_upper_dev_unlink()
7474 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); in netdev_upper_dev_unlink()
7511 struct net_device *dev, in netdev_adjacent_change_prepare() argument
7520 netdev_adjacent_dev_disable(dev, old_dev); in netdev_adjacent_change_prepare()
7522 err = netdev_upper_dev_link(new_dev, dev, extack); in netdev_adjacent_change_prepare()
7525 netdev_adjacent_dev_enable(dev, old_dev); in netdev_adjacent_change_prepare()
7535 struct net_device *dev) in netdev_adjacent_change_commit() argument
7543 netdev_adjacent_dev_enable(dev, old_dev); in netdev_adjacent_change_commit()
7544 netdev_upper_dev_unlink(old_dev, dev); in netdev_adjacent_change_commit()
7550 struct net_device *dev) in netdev_adjacent_change_abort() argument
7556 netdev_adjacent_dev_enable(dev, old_dev); in netdev_adjacent_change_abort()
7558 netdev_upper_dev_unlink(new_dev, dev); in netdev_adjacent_change_abort()
7570 void netdev_bonding_info_change(struct net_device *dev, in netdev_bonding_info_change() argument
7574 .info.dev = dev, in netdev_bonding_info_change()
7584 static void netdev_adjacent_add_links(struct net_device *dev) in netdev_adjacent_add_links() argument
7588 struct net *net = dev_net(dev); in netdev_adjacent_add_links()
7590 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_add_links()
7591 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
7593 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
7594 &iter->dev->adj_list.lower); in netdev_adjacent_add_links()
7595 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
7596 &dev->adj_list.upper); in netdev_adjacent_add_links()
7599 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_add_links()
7600 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
7602 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
7603 &iter->dev->adj_list.upper); in netdev_adjacent_add_links()
7604 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
7605 &dev->adj_list.lower); in netdev_adjacent_add_links()
7609 static void netdev_adjacent_del_links(struct net_device *dev) in netdev_adjacent_del_links() argument
7613 struct net *net = dev_net(dev); in netdev_adjacent_del_links()
7615 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_del_links()
7616 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
7618 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
7619 &iter->dev->adj_list.lower); in netdev_adjacent_del_links()
7620 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
7621 &dev->adj_list.upper); in netdev_adjacent_del_links()
7624 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_del_links()
7625 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
7627 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
7628 &iter->dev->adj_list.upper); in netdev_adjacent_del_links()
7629 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
7630 &dev->adj_list.lower); in netdev_adjacent_del_links()
7634 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) in netdev_adjacent_rename_links() argument
7638 struct net *net = dev_net(dev); in netdev_adjacent_rename_links()
7640 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_rename_links()
7641 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
7643 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
7644 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
7645 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
7646 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
7649 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_rename_links()
7650 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
7652 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
7653 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
7654 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
7655 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
7659 void *netdev_lower_dev_get_private(struct net_device *dev, in netdev_lower_dev_get_private() argument
7666 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); in netdev_lower_dev_get_private()
7687 .info.dev = lower_dev, in netdev_lower_state_changed()
7697 static void dev_change_rx_flags(struct net_device *dev, int flags) in dev_change_rx_flags() argument
7699 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_rx_flags()
7702 ops->ndo_change_rx_flags(dev, flags); in dev_change_rx_flags()
7705 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) in __dev_set_promiscuity() argument
7707 unsigned int old_flags = dev->flags; in __dev_set_promiscuity()
7713 dev->flags |= IFF_PROMISC; in __dev_set_promiscuity()
7714 dev->promiscuity += inc; in __dev_set_promiscuity()
7715 if (dev->promiscuity == 0) { in __dev_set_promiscuity()
7721 dev->flags &= ~IFF_PROMISC; in __dev_set_promiscuity()
7723 dev->promiscuity -= inc; in __dev_set_promiscuity()
7725 dev->name); in __dev_set_promiscuity()
7729 if (dev->flags != old_flags) { in __dev_set_promiscuity()
7731 dev->name, in __dev_set_promiscuity()
7732 dev->flags & IFF_PROMISC ? "entered" : "left"); in __dev_set_promiscuity()
7738 dev->name, (dev->flags & IFF_PROMISC), in __dev_set_promiscuity()
7746 dev_change_rx_flags(dev, IFF_PROMISC); in __dev_set_promiscuity()
7749 __dev_notify_flags(dev, old_flags, IFF_PROMISC); in __dev_set_promiscuity()
7764 int dev_set_promiscuity(struct net_device *dev, int inc) in dev_set_promiscuity() argument
7766 unsigned int old_flags = dev->flags; in dev_set_promiscuity()
7769 err = __dev_set_promiscuity(dev, inc, true); in dev_set_promiscuity()
7772 if (dev->flags != old_flags) in dev_set_promiscuity()
7773 dev_set_rx_mode(dev); in dev_set_promiscuity()
7778 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) in __dev_set_allmulti() argument
7780 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; in __dev_set_allmulti()
7784 dev->flags |= IFF_ALLMULTI; in __dev_set_allmulti()
7785 dev->allmulti += inc; in __dev_set_allmulti()
7786 if (dev->allmulti == 0) { in __dev_set_allmulti()
7792 dev->flags &= ~IFF_ALLMULTI; in __dev_set_allmulti()
7794 dev->allmulti -= inc; in __dev_set_allmulti()
7796 dev->name); in __dev_set_allmulti()
7800 if (dev->flags ^ old_flags) { in __dev_set_allmulti()
7801 dev_change_rx_flags(dev, IFF_ALLMULTI); in __dev_set_allmulti()
7802 dev_set_rx_mode(dev); in __dev_set_allmulti()
7804 __dev_notify_flags(dev, old_flags, in __dev_set_allmulti()
7805 dev->gflags ^ old_gflags); in __dev_set_allmulti()
7823 int dev_set_allmulti(struct net_device *dev, int inc) in dev_set_allmulti() argument
7825 return __dev_set_allmulti(dev, inc, true); in dev_set_allmulti()
7835 void __dev_set_rx_mode(struct net_device *dev) in __dev_set_rx_mode() argument
7837 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_rx_mode()
7840 if (!(dev->flags&IFF_UP)) in __dev_set_rx_mode()
7843 if (!netif_device_present(dev)) in __dev_set_rx_mode()
7846 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { in __dev_set_rx_mode()
7850 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { in __dev_set_rx_mode()
7851 __dev_set_promiscuity(dev, 1, false); in __dev_set_rx_mode()
7852 dev->uc_promisc = true; in __dev_set_rx_mode()
7853 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { in __dev_set_rx_mode()
7854 __dev_set_promiscuity(dev, -1, false); in __dev_set_rx_mode()
7855 dev->uc_promisc = false; in __dev_set_rx_mode()
7860 ops->ndo_set_rx_mode(dev); in __dev_set_rx_mode()
7863 void dev_set_rx_mode(struct net_device *dev) in dev_set_rx_mode() argument
7865 netif_addr_lock_bh(dev); in dev_set_rx_mode()
7866 __dev_set_rx_mode(dev); in dev_set_rx_mode()
7867 netif_addr_unlock_bh(dev); in dev_set_rx_mode()
7876 unsigned int dev_get_flags(const struct net_device *dev) in dev_get_flags() argument
7880 flags = (dev->flags & ~(IFF_PROMISC | in dev_get_flags()
7885 (dev->gflags & (IFF_PROMISC | in dev_get_flags()
7888 if (netif_running(dev)) { in dev_get_flags()
7889 if (netif_oper_up(dev)) in dev_get_flags()
7891 if (netif_carrier_ok(dev)) in dev_get_flags()
7893 if (netif_dormant(dev)) in dev_get_flags()
7901 int __dev_change_flags(struct net_device *dev, unsigned int flags, in __dev_change_flags() argument
7904 unsigned int old_flags = dev->flags; in __dev_change_flags()
7913 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | in __dev_change_flags()
7916 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | in __dev_change_flags()
7924 dev_change_rx_flags(dev, IFF_MULTICAST); in __dev_change_flags()
7926 dev_set_rx_mode(dev); in __dev_change_flags()
7937 __dev_close(dev); in __dev_change_flags()
7939 ret = __dev_open(dev, extack); in __dev_change_flags()
7942 if ((flags ^ dev->gflags) & IFF_PROMISC) { in __dev_change_flags()
7944 unsigned int old_flags = dev->flags; in __dev_change_flags()
7946 dev->gflags ^= IFF_PROMISC; in __dev_change_flags()
7948 if (__dev_set_promiscuity(dev, inc, false) >= 0) in __dev_change_flags()
7949 if (dev->flags != old_flags) in __dev_change_flags()
7950 dev_set_rx_mode(dev); in __dev_change_flags()
7957 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { in __dev_change_flags()
7960 dev->gflags ^= IFF_ALLMULTI; in __dev_change_flags()
7961 __dev_set_allmulti(dev, inc, false); in __dev_change_flags()
7967 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, in __dev_notify_flags() argument
7970 unsigned int changes = dev->flags ^ old_flags; in __dev_notify_flags()
7973 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); in __dev_notify_flags()
7976 if (dev->flags & IFF_UP) in __dev_notify_flags()
7977 call_netdevice_notifiers(NETDEV_UP, dev); in __dev_notify_flags()
7979 call_netdevice_notifiers(NETDEV_DOWN, dev); in __dev_notify_flags()
7982 if (dev->flags & IFF_UP && in __dev_notify_flags()
7986 .dev = dev, in __dev_notify_flags()
8004 int dev_change_flags(struct net_device *dev, unsigned int flags, in dev_change_flags() argument
8008 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; in dev_change_flags()
8010 ret = __dev_change_flags(dev, flags, extack); in dev_change_flags()
8014 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); in dev_change_flags()
8015 __dev_notify_flags(dev, old_flags, changes); in dev_change_flags()
8020 int __dev_set_mtu(struct net_device *dev, int new_mtu) in __dev_set_mtu() argument
8022 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_mtu()
8025 return ops->ndo_change_mtu(dev, new_mtu); in __dev_set_mtu()
8028 WRITE_ONCE(dev->mtu, new_mtu); in __dev_set_mtu()
8033 int dev_validate_mtu(struct net_device *dev, int new_mtu, in dev_validate_mtu() argument
8037 if (new_mtu < 0 || new_mtu < dev->min_mtu) { in dev_validate_mtu()
8042 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { in dev_validate_mtu()
8057 int dev_set_mtu_ext(struct net_device *dev, int new_mtu, in dev_set_mtu_ext() argument
8062 if (new_mtu == dev->mtu) in dev_set_mtu_ext()
8065 err = dev_validate_mtu(dev, new_mtu, extack); in dev_set_mtu_ext()
8069 if (!netif_device_present(dev)) in dev_set_mtu_ext()
8072 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); in dev_set_mtu_ext()
8077 orig_mtu = dev->mtu; in dev_set_mtu_ext()
8078 err = __dev_set_mtu(dev, new_mtu); in dev_set_mtu_ext()
8081 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, in dev_set_mtu_ext()
8088 __dev_set_mtu(dev, orig_mtu); in dev_set_mtu_ext()
8089 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, in dev_set_mtu_ext()
8096 int dev_set_mtu(struct net_device *dev, int new_mtu) in dev_set_mtu() argument
8102 err = dev_set_mtu_ext(dev, new_mtu, &extack); in dev_set_mtu()
8104 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); in dev_set_mtu()
8114 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len) in dev_change_tx_queue_len() argument
8116 unsigned int orig_len = dev->tx_queue_len; in dev_change_tx_queue_len()
8123 dev->tx_queue_len = new_len; in dev_change_tx_queue_len()
8124 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev); in dev_change_tx_queue_len()
8128 res = dev_qdisc_change_tx_queue_len(dev); in dev_change_tx_queue_len()
8136 netdev_err(dev, "refused to change device tx_queue_len\n"); in dev_change_tx_queue_len()
8137 dev->tx_queue_len = orig_len; in dev_change_tx_queue_len()
8146 void dev_set_group(struct net_device *dev, int new_group) in dev_set_group() argument
8148 dev->group = new_group; in dev_set_group()
8158 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr, in dev_pre_changeaddr_notify() argument
8162 .info.dev = dev, in dev_pre_changeaddr_notify()
8181 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, in dev_set_mac_address() argument
8184 const struct net_device_ops *ops = dev->netdev_ops; in dev_set_mac_address()
8189 if (sa->sa_family != dev->type) in dev_set_mac_address()
8191 if (!netif_device_present(dev)) in dev_set_mac_address()
8193 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack); in dev_set_mac_address()
8196 err = ops->ndo_set_mac_address(dev, sa); in dev_set_mac_address()
8199 dev->addr_assign_type = NET_ADDR_SET; in dev_set_mac_address()
8200 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); in dev_set_mac_address()
8201 add_device_randomness(dev->dev_addr, dev->addr_len); in dev_set_mac_address()
8208 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa, in dev_set_mac_address_user() argument
8214 ret = dev_set_mac_address(dev, sa, extack); in dev_set_mac_address_user()
8223 struct net_device *dev; in dev_get_mac_address() local
8229 dev = dev_get_by_name_rcu(net, dev_name); in dev_get_mac_address()
8230 if (!dev) { in dev_get_mac_address()
8234 if (!dev->addr_len) in dev_get_mac_address()
8237 memcpy(sa->sa_data, dev->dev_addr, in dev_get_mac_address()
8238 min_t(size_t, size, dev->addr_len)); in dev_get_mac_address()
8239 sa->sa_family = dev->type; in dev_get_mac_address()
8255 int dev_change_carrier(struct net_device *dev, bool new_carrier) in dev_change_carrier() argument
8257 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_carrier()
8261 if (!netif_device_present(dev)) in dev_change_carrier()
8263 return ops->ndo_change_carrier(dev, new_carrier); in dev_change_carrier()
8274 int dev_get_phys_port_id(struct net_device *dev, in dev_get_phys_port_id() argument
8277 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_id()
8281 return ops->ndo_get_phys_port_id(dev, ppid); in dev_get_phys_port_id()
8293 int dev_get_phys_port_name(struct net_device *dev, in dev_get_phys_port_name() argument
8296 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_name()
8300 err = ops->ndo_get_phys_port_name(dev, name, len); in dev_get_phys_port_name()
8304 return devlink_compat_phys_port_name_get(dev, name, len); in dev_get_phys_port_name()
8316 int dev_get_port_parent_id(struct net_device *dev, in dev_get_port_parent_id() argument
8320 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_port_parent_id()
8327 err = ops->ndo_get_port_parent_id(dev, ppid); in dev_get_port_parent_id()
8332 err = devlink_compat_switch_id_get(dev, ppid); in dev_get_port_parent_id()
8339 netdev_for_each_lower_dev(dev, lower_dev, iter) { in dev_get_port_parent_id()
8380 int dev_change_proto_down(struct net_device *dev, bool proto_down) in dev_change_proto_down() argument
8382 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_proto_down()
8386 if (!netif_device_present(dev)) in dev_change_proto_down()
8388 return ops->ndo_change_proto_down(dev, proto_down); in dev_change_proto_down()
8400 int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) in dev_change_proto_down_generic() argument
8403 netif_carrier_off(dev); in dev_change_proto_down_generic()
8405 netif_carrier_on(dev); in dev_change_proto_down_generic()
8406 dev->proto_down = proto_down; in dev_change_proto_down_generic()
8411 u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op, in __dev_xdp_query() argument
8423 WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG); in __dev_xdp_query()
8428 static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op, in dev_xdp_install() argument
8443 return bpf_op(dev, &xdp); in dev_xdp_install()
8446 static void dev_xdp_uninstall(struct net_device *dev) in dev_xdp_uninstall() argument
8452 WARN_ON(dev_xdp_install(dev, generic_xdp_install, NULL, 0, NULL)); in dev_xdp_uninstall()
8455 ndo_bpf = dev->netdev_ops->ndo_bpf; in dev_xdp_uninstall()
8461 WARN_ON(ndo_bpf(dev, &xdp)); in dev_xdp_uninstall()
8463 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, in dev_xdp_uninstall()
8469 if (!ndo_bpf(dev, &xdp) && xdp.prog_id) in dev_xdp_uninstall()
8470 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, in dev_xdp_uninstall()
8483 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, in dev_change_xdp_fd() argument
8486 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_xdp_fd()
8511 if (!offload && __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG)) { in dev_change_xdp_fd()
8516 prog_id = __dev_xdp_query(dev, bpf_op, query); in dev_change_xdp_fd()
8539 if (!__dev_xdp_query(dev, bpf_op, query)) in dev_change_xdp_fd()
8543 err = dev_xdp_install(dev, bpf_op, extack, flags, prog); in dev_change_xdp_fd()
8574 static void net_set_todo(struct net_device *dev) in net_set_todo() argument
8576 list_add_tail(&dev->todo_list, &net_todo_list); in net_set_todo()
8577 dev_net(dev)->dev_unreg_count++; in net_set_todo()
8582 struct net_device *dev, *tmp; in rollback_registered_many() local
8588 list_for_each_entry_safe(dev, tmp, head, unreg_list) { in rollback_registered_many()
8593 if (dev->reg_state == NETREG_UNINITIALIZED) { in rollback_registered_many()
8595 dev->name, dev); in rollback_registered_many()
8598 list_del(&dev->unreg_list); in rollback_registered_many()
8601 dev->dismantle = true; in rollback_registered_many()
8602 BUG_ON(dev->reg_state != NETREG_REGISTERED); in rollback_registered_many()
8606 list_for_each_entry(dev, head, unreg_list) in rollback_registered_many()
8607 list_add_tail(&dev->close_list, &close_head); in rollback_registered_many()
8610 list_for_each_entry(dev, head, unreg_list) { in rollback_registered_many()
8612 unlist_netdevice(dev); in rollback_registered_many()
8614 dev->reg_state = NETREG_UNREGISTERING; in rollback_registered_many()
8620 list_for_each_entry(dev, head, unreg_list) { in rollback_registered_many()
8624 dev_shutdown(dev); in rollback_registered_many()
8626 dev_xdp_uninstall(dev); in rollback_registered_many()
8631 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); in rollback_registered_many()
8633 if (!dev->rtnl_link_ops || in rollback_registered_many()
8634 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) in rollback_registered_many()
8635 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0, in rollback_registered_many()
8641 dev_uc_flush(dev); in rollback_registered_many()
8642 dev_mc_flush(dev); in rollback_registered_many()
8644 if (dev->netdev_ops->ndo_uninit) in rollback_registered_many()
8645 dev->netdev_ops->ndo_uninit(dev); in rollback_registered_many()
8648 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); in rollback_registered_many()
8651 WARN_ON(netdev_has_any_upper_dev(dev)); in rollback_registered_many()
8652 WARN_ON(netdev_has_any_lower_dev(dev)); in rollback_registered_many()
8655 netdev_unregister_kobject(dev); in rollback_registered_many()
8658 netif_reset_xps_queues_gt(dev, 0); in rollback_registered_many()
8664 list_for_each_entry(dev, head, unreg_list) in rollback_registered_many()
8665 dev_put(dev); in rollback_registered_many()
8668 static void rollback_registered(struct net_device *dev) in rollback_registered() argument
8672 list_add(&dev->unreg_list, &single); in rollback_registered()
8721 static netdev_features_t netdev_fix_features(struct net_device *dev, in netdev_fix_features() argument
8727 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); in netdev_fix_features()
8733 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); in netdev_fix_features()
8739 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); in netdev_fix_features()
8746 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); in netdev_fix_features()
8760 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); in netdev_fix_features()
8765 if ((features & dev->gso_partial_features) && in netdev_fix_features()
8767 netdev_dbg(dev, in netdev_fix_features()
8769 features &= ~dev->gso_partial_features; in netdev_fix_features()
8779 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n"); in netdev_fix_features()
8787 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
8792 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
8798 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n"); in netdev_fix_features()
8805 int __netdev_update_features(struct net_device *dev) in __netdev_update_features() argument
8814 features = netdev_get_wanted_features(dev); in __netdev_update_features()
8816 if (dev->netdev_ops->ndo_fix_features) in __netdev_update_features()
8817 features = dev->netdev_ops->ndo_fix_features(dev, features); in __netdev_update_features()
8820 features = netdev_fix_features(dev, features); in __netdev_update_features()
8823 netdev_for_each_upper_dev_rcu(dev, upper, iter) in __netdev_update_features()
8824 features = netdev_sync_upper_features(dev, upper, features); in __netdev_update_features()
8826 if (dev->features == features) in __netdev_update_features()
8829 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", in __netdev_update_features()
8830 &dev->features, &features); in __netdev_update_features()
8832 if (dev->netdev_ops->ndo_set_features) in __netdev_update_features()
8833 err = dev->netdev_ops->ndo_set_features(dev, features); in __netdev_update_features()
8838 netdev_err(dev, in __netdev_update_features()
8840 err, &features, &dev->features); in __netdev_update_features()
8851 netdev_for_each_lower_dev(dev, lower, iter) in __netdev_update_features()
8852 netdev_sync_lower_features(dev, lower, features); in __netdev_update_features()
8855 netdev_features_t diff = features ^ dev->features; in __netdev_update_features()
8866 dev->features = features; in __netdev_update_features()
8867 udp_tunnel_get_rx_info(dev); in __netdev_update_features()
8869 udp_tunnel_drop_rx_info(dev); in __netdev_update_features()
8875 dev->features = features; in __netdev_update_features()
8876 err |= vlan_get_rx_ctag_filter_info(dev); in __netdev_update_features()
8878 vlan_drop_rx_ctag_filter_info(dev); in __netdev_update_features()
8884 dev->features = features; in __netdev_update_features()
8885 err |= vlan_get_rx_stag_filter_info(dev); in __netdev_update_features()
8887 vlan_drop_rx_stag_filter_info(dev); in __netdev_update_features()
8891 dev->features = features; in __netdev_update_features()
8905 void netdev_update_features(struct net_device *dev) in netdev_update_features() argument
8907 if (__netdev_update_features(dev)) in netdev_update_features()
8908 netdev_features_change(dev); in netdev_update_features()
8922 void netdev_change_features(struct net_device *dev) in netdev_change_features() argument
8924 __netdev_update_features(dev); in netdev_change_features()
8925 netdev_features_change(dev); in netdev_change_features()
8939 struct net_device *dev) in netif_stacked_transfer_operstate() argument
8942 netif_dormant_on(dev); in netif_stacked_transfer_operstate()
8944 netif_dormant_off(dev); in netif_stacked_transfer_operstate()
8947 netif_carrier_on(dev); in netif_stacked_transfer_operstate()
8949 netif_carrier_off(dev); in netif_stacked_transfer_operstate()
8953 static int netif_alloc_rx_queues(struct net_device *dev) in netif_alloc_rx_queues() argument
8955 unsigned int i, count = dev->num_rx_queues; in netif_alloc_rx_queues()
8966 dev->_rx = rx; in netif_alloc_rx_queues()
8969 rx[i].dev = dev; in netif_alloc_rx_queues()
8972 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); in netif_alloc_rx_queues()
8982 kvfree(dev->_rx); in netif_alloc_rx_queues()
8983 dev->_rx = NULL; in netif_alloc_rx_queues()
8987 static void netif_free_rx_queues(struct net_device *dev) in netif_free_rx_queues() argument
8989 unsigned int i, count = dev->num_rx_queues; in netif_free_rx_queues()
8992 if (!dev->_rx) in netif_free_rx_queues()
8996 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); in netif_free_rx_queues()
8998 kvfree(dev->_rx); in netif_free_rx_queues()
9001 static void netdev_init_one_queue(struct net_device *dev, in netdev_init_one_queue() argument
9006 lockdep_set_class(&queue->_xmit_lock, &dev->qdisc_xmit_lock_key); in netdev_init_one_queue()
9009 queue->dev = dev; in netdev_init_one_queue()
9015 static void netif_free_tx_queues(struct net_device *dev) in netif_free_tx_queues() argument
9017 kvfree(dev->_tx); in netif_free_tx_queues()
9020 static int netif_alloc_netdev_queues(struct net_device *dev) in netif_alloc_netdev_queues() argument
9022 unsigned int count = dev->num_tx_queues; in netif_alloc_netdev_queues()
9033 dev->_tx = tx; in netif_alloc_netdev_queues()
9035 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); in netif_alloc_netdev_queues()
9036 spin_lock_init(&dev->tx_global_lock); in netif_alloc_netdev_queues()
9041 void netif_tx_stop_all_queues(struct net_device *dev) in netif_tx_stop_all_queues() argument
9045 for (i = 0; i < dev->num_tx_queues; i++) { in netif_tx_stop_all_queues()
9046 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); in netif_tx_stop_all_queues()
9053 static void netdev_register_lockdep_key(struct net_device *dev) in netdev_register_lockdep_key() argument
9055 lockdep_register_key(&dev->qdisc_tx_busylock_key); in netdev_register_lockdep_key()
9056 lockdep_register_key(&dev->qdisc_running_key); in netdev_register_lockdep_key()
9057 lockdep_register_key(&dev->qdisc_xmit_lock_key); in netdev_register_lockdep_key()
9058 lockdep_register_key(&dev->addr_list_lock_key); in netdev_register_lockdep_key()
9061 static void netdev_unregister_lockdep_key(struct net_device *dev) in netdev_unregister_lockdep_key() argument
9063 lockdep_unregister_key(&dev->qdisc_tx_busylock_key); in netdev_unregister_lockdep_key()
9064 lockdep_unregister_key(&dev->qdisc_running_key); in netdev_unregister_lockdep_key()
9065 lockdep_unregister_key(&dev->qdisc_xmit_lock_key); in netdev_unregister_lockdep_key()
9066 lockdep_unregister_key(&dev->addr_list_lock_key); in netdev_unregister_lockdep_key()
9069 void netdev_update_lockdep_key(struct net_device *dev) in netdev_update_lockdep_key() argument
9071 lockdep_unregister_key(&dev->addr_list_lock_key); in netdev_update_lockdep_key()
9072 lockdep_register_key(&dev->addr_list_lock_key); in netdev_update_lockdep_key()
9074 lockdep_set_class(&dev->addr_list_lock, &dev->addr_list_lock_key); in netdev_update_lockdep_key()
9095 int register_netdevice(struct net_device *dev) in register_netdevice() argument
9098 struct net *net = dev_net(dev); in register_netdevice()
9108 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); in register_netdevice()
9111 spin_lock_init(&dev->addr_list_lock); in register_netdevice()
9112 lockdep_set_class(&dev->addr_list_lock, &dev->addr_list_lock_key); in register_netdevice()
9114 ret = dev_get_valid_name(net, dev, dev->name); in register_netdevice()
9119 if (dev->netdev_ops->ndo_init) { in register_netdevice()
9120 ret = dev->netdev_ops->ndo_init(dev); in register_netdevice()
9128 if (((dev->hw_features | dev->features) & in register_netdevice()
9130 (!dev->netdev_ops->ndo_vlan_rx_add_vid || in register_netdevice()
9131 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { in register_netdevice()
9132 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); in register_netdevice()
9138 if (!dev->ifindex) in register_netdevice()
9139 dev->ifindex = dev_new_index(net); in register_netdevice()
9140 else if (__dev_get_by_index(net, dev->ifindex)) in register_netdevice()
9146 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); in register_netdevice()
9147 dev->features |= NETIF_F_SOFT_FEATURES; in register_netdevice()
9149 if (dev->netdev_ops->ndo_udp_tunnel_add) { in register_netdevice()
9150 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
9151 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
9154 dev->wanted_features = dev->features & dev->hw_features; in register_netdevice()
9156 if (!(dev->flags & IFF_LOOPBACK)) in register_netdevice()
9157 dev->hw_features |= NETIF_F_NOCACHE_COPY; in register_netdevice()
9164 if (dev->hw_features & NETIF_F_TSO) in register_netdevice()
9165 dev->hw_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
9166 if (dev->vlan_features & NETIF_F_TSO) in register_netdevice()
9167 dev->vlan_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
9168 if (dev->mpls_features & NETIF_F_TSO) in register_netdevice()
9169 dev->mpls_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
9170 if (dev->hw_enc_features & NETIF_F_TSO) in register_netdevice()
9171 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
9175 dev->vlan_features |= NETIF_F_HIGHDMA; in register_netdevice()
9179 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; in register_netdevice()
9183 dev->mpls_features |= NETIF_F_SG; in register_netdevice()
9185 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); in register_netdevice()
9190 ret = netdev_register_kobject(dev); in register_netdevice()
9192 dev->reg_state = NETREG_UNREGISTERED; in register_netdevice()
9195 dev->reg_state = NETREG_REGISTERED; in register_netdevice()
9197 __netdev_update_features(dev); in register_netdevice()
9204 set_bit(__LINK_STATE_PRESENT, &dev->state); in register_netdevice()
9206 linkwatch_init_dev(dev); in register_netdevice()
9208 dev_init_scheduler(dev); in register_netdevice()
9209 dev_hold(dev); in register_netdevice()
9210 list_netdevice(dev); in register_netdevice()
9211 add_device_randomness(dev->dev_addr, dev->addr_len); in register_netdevice()
9217 if (dev->addr_assign_type == NET_ADDR_PERM) in register_netdevice()
9218 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); in register_netdevice()
9221 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); in register_netdevice()
9224 rollback_registered(dev); in register_netdevice()
9227 dev->reg_state = NETREG_UNREGISTERED; in register_netdevice()
9234 kobject_put(&dev->dev.kobj); in register_netdevice()
9240 if (!dev->rtnl_link_ops || in register_netdevice()
9241 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) in register_netdevice()
9242 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); in register_netdevice()
9248 if (dev->netdev_ops->ndo_uninit) in register_netdevice()
9249 dev->netdev_ops->ndo_uninit(dev); in register_netdevice()
9250 if (dev->priv_destructor) in register_netdevice()
9251 dev->priv_destructor(dev); in register_netdevice()
9266 int init_dummy_netdev(struct net_device *dev) in init_dummy_netdev() argument
9273 memset(dev, 0, sizeof(struct net_device)); in init_dummy_netdev()
9278 dev->reg_state = NETREG_DUMMY; in init_dummy_netdev()
9281 INIT_LIST_HEAD(&dev->napi_list); in init_dummy_netdev()
9284 set_bit(__LINK_STATE_PRESENT, &dev->state); in init_dummy_netdev()
9285 set_bit(__LINK_STATE_START, &dev->state); in init_dummy_netdev()
9288 dev_net_set(dev, &init_net); in init_dummy_netdev()
9313 int register_netdev(struct net_device *dev) in register_netdev() argument
9319 err = register_netdevice(dev); in register_netdev()
9325 int netdev_refcnt_read(const struct net_device *dev) in netdev_refcnt_read() argument
9330 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); in netdev_refcnt_read()
9347 static void netdev_wait_allrefs(struct net_device *dev) in netdev_wait_allrefs() argument
9352 linkwatch_forget_dev(dev); in netdev_wait_allrefs()
9355 refcnt = netdev_refcnt_read(dev); in netdev_wait_allrefs()
9362 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); in netdev_wait_allrefs()
9369 &dev->state)) { in netdev_wait_allrefs()
9386 refcnt = netdev_refcnt_read(dev); in netdev_wait_allrefs()
9390 dev->name, refcnt); in netdev_wait_allrefs()
9435 struct net_device *dev in netdev_run_todo() local
9437 list_del(&dev->todo_list); in netdev_run_todo()
9439 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { in netdev_run_todo()
9441 dev->name, dev->reg_state); in netdev_run_todo()
9446 dev->reg_state = NETREG_UNREGISTERED; in netdev_run_todo()
9448 netdev_wait_allrefs(dev); in netdev_run_todo()
9451 BUG_ON(netdev_refcnt_read(dev)); in netdev_run_todo()
9452 BUG_ON(!list_empty(&dev->ptype_all)); in netdev_run_todo()
9453 BUG_ON(!list_empty(&dev->ptype_specific)); in netdev_run_todo()
9454 WARN_ON(rcu_access_pointer(dev->ip_ptr)); in netdev_run_todo()
9455 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); in netdev_run_todo()
9457 if (dev->priv_destructor) in netdev_run_todo()
9458 dev->priv_destructor(dev); in netdev_run_todo()
9459 if (dev->needs_free_netdev) in netdev_run_todo()
9460 free_netdev(dev); in netdev_run_todo()
9464 dev_net(dev)->dev_unreg_count--; in netdev_run_todo()
9469 kobject_put(&dev->dev.kobj); in netdev_run_todo()
9512 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, in dev_get_stats() argument
9515 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_stats()
9519 ops->ndo_get_stats64(dev, storage); in dev_get_stats()
9521 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); in dev_get_stats()
9523 netdev_stats_to_stats64(storage, &dev->stats); in dev_get_stats()
9525 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped); in dev_get_stats()
9526 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped); in dev_get_stats()
9527 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler); in dev_get_stats()
9532 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) in dev_ingress_queue_create() argument
9534 struct netdev_queue *queue = dev_ingress_queue(dev); in dev_ingress_queue_create()
9542 netdev_init_one_queue(dev, queue, NULL); in dev_ingress_queue_create()
9545 rcu_assign_pointer(dev->ingress_queue, queue); in dev_ingress_queue_create()
9552 void netdev_set_default_ethtool_ops(struct net_device *dev, in netdev_set_default_ethtool_ops() argument
9555 if (dev->ethtool_ops == &default_ethtool_ops) in netdev_set_default_ethtool_ops()
9556 dev->ethtool_ops = ops; in netdev_set_default_ethtool_ops()
9560 void netdev_freemem(struct net_device *dev) in netdev_freemem() argument
9562 char *addr = (char *)dev - dev->padded; in netdev_freemem()
9585 struct net_device *dev; in alloc_netdev_mqs() local
9589 BUG_ON(strlen(name) >= sizeof(dev->name)); in alloc_netdev_mqs()
9614 dev = PTR_ALIGN(p, NETDEV_ALIGN); in alloc_netdev_mqs()
9615 dev->padded = (char *)dev - (char *)p; in alloc_netdev_mqs()
9617 dev->pcpu_refcnt = alloc_percpu(int); in alloc_netdev_mqs()
9618 if (!dev->pcpu_refcnt) in alloc_netdev_mqs()
9621 if (dev_addr_init(dev)) in alloc_netdev_mqs()
9624 dev_mc_init(dev); in alloc_netdev_mqs()
9625 dev_uc_init(dev); in alloc_netdev_mqs()
9627 dev_net_set(dev, &init_net); in alloc_netdev_mqs()
9629 netdev_register_lockdep_key(dev); in alloc_netdev_mqs()
9631 dev->gso_max_size = GSO_MAX_SIZE; in alloc_netdev_mqs()
9632 dev->gso_max_segs = GSO_MAX_SEGS; in alloc_netdev_mqs()
9633 dev->upper_level = 1; in alloc_netdev_mqs()
9634 dev->lower_level = 1; in alloc_netdev_mqs()
9636 INIT_LIST_HEAD(&dev->napi_list); in alloc_netdev_mqs()
9637 INIT_LIST_HEAD(&dev->unreg_list); in alloc_netdev_mqs()
9638 INIT_LIST_HEAD(&dev->close_list); in alloc_netdev_mqs()
9639 INIT_LIST_HEAD(&dev->link_watch_list); in alloc_netdev_mqs()
9640 INIT_LIST_HEAD(&dev->adj_list.upper); in alloc_netdev_mqs()
9641 INIT_LIST_HEAD(&dev->adj_list.lower); in alloc_netdev_mqs()
9642 INIT_LIST_HEAD(&dev->ptype_all); in alloc_netdev_mqs()
9643 INIT_LIST_HEAD(&dev->ptype_specific); in alloc_netdev_mqs()
9645 hash_init(dev->qdisc_hash); in alloc_netdev_mqs()
9647 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; in alloc_netdev_mqs()
9648 setup(dev); in alloc_netdev_mqs()
9650 if (!dev->tx_queue_len) { in alloc_netdev_mqs()
9651 dev->priv_flags |= IFF_NO_QUEUE; in alloc_netdev_mqs()
9652 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; in alloc_netdev_mqs()
9655 dev->num_tx_queues = txqs; in alloc_netdev_mqs()
9656 dev->real_num_tx_queues = txqs; in alloc_netdev_mqs()
9657 if (netif_alloc_netdev_queues(dev)) in alloc_netdev_mqs()
9660 dev->num_rx_queues = rxqs; in alloc_netdev_mqs()
9661 dev->real_num_rx_queues = rxqs; in alloc_netdev_mqs()
9662 if (netif_alloc_rx_queues(dev)) in alloc_netdev_mqs()
9665 strcpy(dev->name, name); in alloc_netdev_mqs()
9666 dev->name_assign_type = name_assign_type; in alloc_netdev_mqs()
9667 dev->group = INIT_NETDEV_GROUP; in alloc_netdev_mqs()
9668 if (!dev->ethtool_ops) in alloc_netdev_mqs()
9669 dev->ethtool_ops = &default_ethtool_ops; in alloc_netdev_mqs()
9671 nf_hook_ingress_init(dev); in alloc_netdev_mqs()
9673 return dev; in alloc_netdev_mqs()
9676 free_netdev(dev); in alloc_netdev_mqs()
9680 free_percpu(dev->pcpu_refcnt); in alloc_netdev_mqs()
9682 netdev_freemem(dev); in alloc_netdev_mqs()
9696 void free_netdev(struct net_device *dev) in free_netdev() argument
9701 netif_free_tx_queues(dev); in free_netdev()
9702 netif_free_rx_queues(dev); in free_netdev()
9704 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); in free_netdev()
9707 dev_addr_flush(dev); in free_netdev()
9709 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) in free_netdev()
9712 free_percpu(dev->pcpu_refcnt); in free_netdev()
9713 dev->pcpu_refcnt = NULL; in free_netdev()
9715 netdev_unregister_lockdep_key(dev); in free_netdev()
9718 if (dev->reg_state == NETREG_UNINITIALIZED) { in free_netdev()
9719 netdev_freemem(dev); in free_netdev()
9723 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); in free_netdev()
9724 dev->reg_state = NETREG_RELEASED; in free_netdev()
9727 put_device(&dev->dev); in free_netdev()
9760 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) in unregister_netdevice_queue() argument
9765 list_move_tail(&dev->unreg_list, head); in unregister_netdevice_queue()
9767 rollback_registered(dev); in unregister_netdevice_queue()
9769 net_set_todo(dev); in unregister_netdevice_queue()
9783 struct net_device *dev; in unregister_netdevice_many() local
9787 list_for_each_entry(dev, head, unreg_list) in unregister_netdevice_many()
9788 net_set_todo(dev); in unregister_netdevice_many()
9805 void unregister_netdev(struct net_device *dev) in unregister_netdev() argument
9808 unregister_netdevice(dev); in unregister_netdev()
9827 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) in dev_change_net_namespace() argument
9835 if (dev->features & NETIF_F_NETNS_LOCAL) in dev_change_net_namespace()
9839 if (dev->reg_state != NETREG_REGISTERED) in dev_change_net_namespace()
9844 if (net_eq(dev_net(dev), net)) in dev_change_net_namespace()
9851 if (__dev_get_by_name(net, dev->name)) { in dev_change_net_namespace()
9855 err = dev_get_valid_name(net, dev, pat); in dev_change_net_namespace()
9865 dev_close(dev); in dev_change_net_namespace()
9868 unlist_netdevice(dev); in dev_change_net_namespace()
9873 dev_shutdown(dev); in dev_change_net_namespace()
9882 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); in dev_change_net_namespace()
9885 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL); in dev_change_net_namespace()
9887 if (__dev_get_by_index(net, dev->ifindex)) in dev_change_net_namespace()
9890 new_ifindex = dev->ifindex; in dev_change_net_namespace()
9892 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid, in dev_change_net_namespace()
9898 dev_uc_flush(dev); in dev_change_net_namespace()
9899 dev_mc_flush(dev); in dev_change_net_namespace()
9902 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); in dev_change_net_namespace()
9903 netdev_adjacent_del_links(dev); in dev_change_net_namespace()
9906 dev_net_set(dev, net); in dev_change_net_namespace()
9907 dev->ifindex = new_ifindex; in dev_change_net_namespace()
9910 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); in dev_change_net_namespace()
9911 netdev_adjacent_add_links(dev); in dev_change_net_namespace()
9914 err = device_rename(&dev->dev, dev->name); in dev_change_net_namespace()
9918 list_netdevice(dev); in dev_change_net_namespace()
9921 call_netdevice_notifiers(NETDEV_REGISTER, dev); in dev_change_net_namespace()
9927 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); in dev_change_net_namespace()
10074 const char *netdev_drivername(const struct net_device *dev) in netdev_drivername() argument
10080 parent = dev->dev.parent; in netdev_drivername()
10090 static void __netdev_printk(const char *level, const struct net_device *dev, in __netdev_printk() argument
10093 if (dev && dev->dev.parent) { in __netdev_printk()
10095 dev->dev.parent, in __netdev_printk()
10097 dev_driver_string(dev->dev.parent), in __netdev_printk()
10098 dev_name(dev->dev.parent), in __netdev_printk()
10099 netdev_name(dev), netdev_reg_state(dev), in __netdev_printk()
10101 } else if (dev) { in __netdev_printk()
10103 level, netdev_name(dev), netdev_reg_state(dev), vaf); in __netdev_printk()
10109 void netdev_printk(const char *level, const struct net_device *dev, in netdev_printk() argument
10120 __netdev_printk(level, dev, &vaf); in netdev_printk()
10127 void func(const struct net_device *dev, const char *fmt, ...) \
10137 __netdev_printk(level, dev, &vaf); \
10166 struct net_device *dev, *aux; in default_device_exit() local
10172 for_each_netdev_safe(net, dev, aux) { in default_device_exit()
10177 if (dev->features & NETIF_F_NETNS_LOCAL) in default_device_exit()
10181 if (dev->rtnl_link_ops) in default_device_exit()
10185 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); in default_device_exit()
10188 err = dev_change_net_namespace(dev, &init_net, fb_name); in default_device_exit()
10191 __func__, dev->name, err); in default_device_exit()
10233 struct net_device *dev; in default_device_exit_batch() local
10250 for_each_netdev_reverse(net, dev) { in default_device_exit_batch()
10251 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) in default_device_exit_batch()
10252 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); in default_device_exit_batch()
10254 unregister_netdevice_queue(dev, &dev_kill_list); in default_device_exit_batch()