Lines Matching refs:napi
940 struct napi_struct *napi; in dev_get_by_napi_id() local
947 napi = napi_by_id(napi_id); in dev_get_by_napi_id()
949 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
4357 struct napi_struct *napi) in ____napi_schedule() argument
4361 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { in ____napi_schedule()
4368 thread = READ_ONCE(napi->thread); in ____napi_schedule()
4376 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in ____napi_schedule()
4382 list_add_tail(&napi->poll_list, &sd->poll_list); in ____napi_schedule()
5876 static void gro_normal_list(struct napi_struct *napi) in gro_normal_list() argument
5878 if (!napi->rx_count) in gro_normal_list()
5880 netif_receive_skb_list_internal(&napi->rx_list); in gro_normal_list()
5881 INIT_LIST_HEAD(&napi->rx_list); in gro_normal_list()
5882 napi->rx_count = 0; in gro_normal_list()
5888 static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) in gro_normal_one() argument
5890 list_add_tail(&skb->list, &napi->rx_list); in gro_normal_one()
5891 napi->rx_count += segs; in gro_normal_one()
5892 if (napi->rx_count >= gro_normal_batch) in gro_normal_one()
5893 gro_normal_list(napi); in gro_normal_one()
5896 static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) in napi_gro_complete() argument
5929 gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count); in napi_gro_complete()
5933 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, in __napi_gro_flush_chain() argument
5936 struct list_head *head = &napi->gro_hash[index].list; in __napi_gro_flush_chain()
5943 napi_gro_complete(napi, skb); in __napi_gro_flush_chain()
5944 napi->gro_hash[index].count--; in __napi_gro_flush_chain()
5947 if (!napi->gro_hash[index].count) in __napi_gro_flush_chain()
5948 __clear_bit(index, &napi->gro_bitmask); in __napi_gro_flush_chain()
5955 void napi_gro_flush(struct napi_struct *napi, bool flush_old) in napi_gro_flush() argument
5957 unsigned long bitmask = napi->gro_bitmask; in napi_gro_flush()
5963 __napi_gro_flush_chain(napi, base, flush_old); in napi_gro_flush()
6067 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) in gro_flush_oldest() argument
6083 napi_gro_complete(napi, oldest); in gro_flush_oldest()
6086 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) in dev_gro_receive() argument
6089 struct gro_list *gro_list = &napi->gro_hash[bucket]; in dev_gro_receive()
6155 napi_gro_complete(napi, pp); in dev_gro_receive()
6166 gro_flush_oldest(napi, &gro_list->list); in dev_gro_receive()
6183 if (!test_bit(bucket, &napi->gro_bitmask)) in dev_gro_receive()
6184 __set_bit(bucket, &napi->gro_bitmask); in dev_gro_receive()
6185 } else if (test_bit(bucket, &napi->gro_bitmask)) { in dev_gro_receive()
6186 __clear_bit(bucket, &napi->gro_bitmask); in dev_gro_receive()
6224 static gro_result_t napi_skb_finish(struct napi_struct *napi, in napi_skb_finish() argument
6230 gro_normal_one(napi, skb, 1); in napi_skb_finish()
6251 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) in napi_gro_receive() argument
6255 skb_mark_napi_id(skb, napi); in napi_gro_receive()
6260 ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); in napi_gro_receive()
6267 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) in napi_reuse_skb() argument
6277 skb->dev = napi->dev; in napi_reuse_skb()
6293 napi->skb = skb; in napi_reuse_skb()
6296 struct sk_buff *napi_get_frags(struct napi_struct *napi) in napi_get_frags() argument
6298 struct sk_buff *skb = napi->skb; in napi_get_frags()
6301 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); in napi_get_frags()
6303 napi->skb = skb; in napi_get_frags()
6304 skb_mark_napi_id(skb, napi); in napi_get_frags()
6311 static gro_result_t napi_frags_finish(struct napi_struct *napi, in napi_frags_finish() argument
6321 gro_normal_one(napi, skb, 1); in napi_frags_finish()
6328 napi_reuse_skb(napi, skb); in napi_frags_finish()
6343 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) in napi_frags_skb() argument
6345 struct sk_buff *skb = napi->skb; in napi_frags_skb()
6349 napi->skb = NULL; in napi_frags_skb()
6358 __func__, napi->dev->name); in napi_frags_skb()
6359 napi_reuse_skb(napi, skb); in napi_frags_skb()
6380 gro_result_t napi_gro_frags(struct napi_struct *napi) in napi_gro_frags() argument
6383 struct sk_buff *skb = napi_frags_skb(napi); in napi_gro_frags()
6387 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); in napi_gro_frags()
6463 static int process_backlog(struct napi_struct *napi, int quota) in process_backlog() argument
6465 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); in process_backlog()
6477 napi->weight = READ_ONCE(dev_rx_weight); in process_backlog()
6502 napi->state = 0; in process_backlog()
6660 struct napi_struct *napi; in napi_by_id() local
6662 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) in napi_by_id()
6663 if (napi->napi_id == napi_id) in napi_by_id()
6664 return napi; in napi_by_id()
6671 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule) in __busy_poll_stop() argument
6674 gro_normal_list(napi); in __busy_poll_stop()
6675 __napi_schedule(napi); in __busy_poll_stop()
6679 if (napi->gro_bitmask) { in __busy_poll_stop()
6683 napi_gro_flush(napi, HZ >= 1000); in __busy_poll_stop()
6686 gro_normal_list(napi); in __busy_poll_stop()
6687 clear_bit(NAPI_STATE_SCHED, &napi->state); in __busy_poll_stop()
6690 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll, in busy_poll_stop() argument
6706 clear_bit(NAPI_STATE_MISSED, &napi->state); in busy_poll_stop()
6707 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); in busy_poll_stop()
6712 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs); in busy_poll_stop()
6713 timeout = READ_ONCE(napi->dev->gro_flush_timeout); in busy_poll_stop()
6714 if (napi->defer_hard_irqs_count && timeout) { in busy_poll_stop()
6715 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED); in busy_poll_stop()
6723 rc = napi->poll(napi, budget); in busy_poll_stop()
6728 trace_napi_poll(napi, rc, budget); in busy_poll_stop()
6731 __busy_poll_stop(napi, skip_schedule); in busy_poll_stop()
6740 int (*napi_poll)(struct napi_struct *napi, int budget); in napi_busy_loop()
6742 struct napi_struct *napi; in napi_busy_loop() local
6749 napi = napi_by_id(napi_id); in napi_busy_loop()
6750 if (!napi) in napi_busy_loop()
6759 unsigned long val = READ_ONCE(napi->state); in napi_busy_loop()
6767 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_busy_loop()
6770 if (cmpxchg(&napi->state, val, in napi_busy_loop()
6774 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_busy_loop()
6777 have_poll_lock = netpoll_poll_lock(napi); in napi_busy_loop()
6778 napi_poll = napi->poll; in napi_busy_loop()
6780 work = napi_poll(napi, budget); in napi_busy_loop()
6781 trace_napi_poll(napi, work, budget); in napi_busy_loop()
6782 gro_normal_list(napi); in napi_busy_loop()
6785 __NET_ADD_STATS(dev_net(napi->dev), in napi_busy_loop()
6794 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget); in napi_busy_loop()
6805 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget); in napi_busy_loop()
6814 static void napi_hash_add(struct napi_struct *napi) in napi_hash_add() argument
6816 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) in napi_hash_add()
6826 napi->napi_id = napi_gen_id; in napi_hash_add()
6828 hlist_add_head_rcu(&napi->napi_hash_node, in napi_hash_add()
6829 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); in napi_hash_add()
6837 static void napi_hash_del(struct napi_struct *napi) in napi_hash_del() argument
6841 hlist_del_init_rcu(&napi->napi_hash_node); in napi_hash_del()
6848 struct napi_struct *napi; in napi_watchdog() local
6850 napi = container_of(timer, struct napi_struct, timer); in napi_watchdog()
6855 if (!napi_disable_pending(napi) && in napi_watchdog()
6856 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { in napi_watchdog()
6857 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_watchdog()
6858 __napi_schedule_irqoff(napi); in napi_watchdog()
6864 static void init_gro_hash(struct napi_struct *napi) in init_gro_hash() argument
6869 INIT_LIST_HEAD(&napi->gro_hash[i].list); in init_gro_hash()
6870 napi->gro_hash[i].count = 0; in init_gro_hash()
6872 napi->gro_bitmask = 0; in init_gro_hash()
6877 struct napi_struct *napi; in dev_set_threaded() local
6884 list_for_each_entry(napi, &dev->napi_list, dev_list) { in dev_set_threaded()
6885 if (!napi->thread) { in dev_set_threaded()
6886 err = napi_kthread_create(napi); in dev_set_threaded()
6908 list_for_each_entry(napi, &dev->napi_list, dev_list) { in dev_set_threaded()
6910 set_bit(NAPI_STATE_THREADED, &napi->state); in dev_set_threaded()
6912 clear_bit(NAPI_STATE_THREADED, &napi->state); in dev_set_threaded()
6919 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, in netif_napi_add() argument
6922 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) in netif_napi_add()
6925 INIT_LIST_HEAD(&napi->poll_list); in netif_napi_add()
6926 INIT_HLIST_NODE(&napi->napi_hash_node); in netif_napi_add()
6927 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in netif_napi_add()
6928 napi->timer.function = napi_watchdog; in netif_napi_add()
6929 init_gro_hash(napi); in netif_napi_add()
6930 napi->skb = NULL; in netif_napi_add()
6931 INIT_LIST_HEAD(&napi->rx_list); in netif_napi_add()
6932 napi->rx_count = 0; in netif_napi_add()
6933 napi->poll = poll; in netif_napi_add()
6937 napi->weight = weight; in netif_napi_add()
6938 napi->dev = dev; in netif_napi_add()
6940 napi->poll_owner = -1; in netif_napi_add()
6942 set_bit(NAPI_STATE_SCHED, &napi->state); in netif_napi_add()
6943 set_bit(NAPI_STATE_NPSVC, &napi->state); in netif_napi_add()
6944 list_add_rcu(&napi->dev_list, &dev->napi_list); in netif_napi_add()
6945 napi_hash_add(napi); in netif_napi_add()
6950 if (dev->threaded && napi_kthread_create(napi)) in netif_napi_add()
6995 static void flush_gro_hash(struct napi_struct *napi) in flush_gro_hash() argument
7002 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list) in flush_gro_hash()
7004 napi->gro_hash[i].count = 0; in flush_gro_hash()
7009 void __netif_napi_del(struct napi_struct *napi) in __netif_napi_del() argument
7011 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) in __netif_napi_del()
7014 napi_hash_del(napi); in __netif_napi_del()
7015 list_del_rcu(&napi->dev_list); in __netif_napi_del()
7016 napi_free_frags(napi); in __netif_napi_del()
7018 flush_gro_hash(napi); in __netif_napi_del()
7019 napi->gro_bitmask = 0; in __netif_napi_del()
7021 if (napi->thread) { in __netif_napi_del()
7022 kthread_stop(napi->thread); in __netif_napi_del()
7023 napi->thread = NULL; in __netif_napi_del()
7119 static int napi_thread_wait(struct napi_struct *napi) in napi_thread_wait() argument
7131 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) { in napi_thread_wait()
7132 WARN_ON(!list_empty(&napi->poll_list)); in napi_thread_wait()
7149 struct napi_struct *napi = data; in napi_threaded_poll() local
7152 while (!napi_thread_wait(napi)) { in napi_threaded_poll()
7158 have = netpoll_poll_lock(napi); in napi_threaded_poll()
7159 __napi_poll(napi, &repoll); in napi_threaded_poll()
11340 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, in dev_cpu_dead() local
11344 list_del_init(&napi->poll_list); in dev_cpu_dead()
11345 if (napi->poll == process_backlog) in dev_cpu_dead()
11346 napi->state = 0; in dev_cpu_dead()
11348 ____napi_schedule(sd, napi); in dev_cpu_dead()