Lines Matching refs:epi
252 struct epitem *epi; member
336 static inline int ep_is_linked(struct epitem *epi) in ep_is_linked() argument
338 return !list_empty(&epi->rdllink); in ep_is_linked()
355 return container_of(p, struct ep_pqueue, pt)->epi; in ep_item_from_epqueue()
416 static inline void ep_set_busy_poll_napi_id(struct epitem *epi) in ep_set_busy_poll_napi_id() argument
427 sock = sock_from_file(epi->ffd.file, &err); in ep_set_busy_poll_napi_id()
436 ep = epi->ep; in ep_set_busy_poll_napi_id()
459 static inline void ep_set_busy_poll_napi_id(struct epitem *epi) in ep_set_busy_poll_napi_id() argument
610 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
612 struct list_head *lsthead = &epi->pwqlist; in ep_unregister_pollwait()
625 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) in ep_wakeup_source() argument
627 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
631 static inline void ep_pm_stay_awake(struct epitem *epi) in ep_pm_stay_awake() argument
633 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_pm_stay_awake()
639 static inline bool ep_has_wakeup_source(struct epitem *epi) in ep_has_wakeup_source() argument
641 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
645 static inline void ep_pm_stay_awake_rcu(struct epitem *epi) in ep_pm_stay_awake_rcu() argument
650 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
676 struct epitem *epi, *nepi; in ep_scan_ready_list() local
713 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; in ep_scan_ready_list()
714 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_scan_ready_list()
721 if (!ep_is_linked(epi)) { in ep_scan_ready_list()
726 list_add(&epi->rdllink, &ep->rdllist); in ep_scan_ready_list()
727 ep_pm_stay_awake(epi); in ep_scan_ready_list()
767 struct epitem *epi = container_of(head, struct epitem, rcu); in epi_rcu_free() local
768 kmem_cache_free(epi_cache, epi); in epi_rcu_free()
775 static int ep_remove(struct eventpoll *ep, struct epitem *epi) in ep_remove() argument
777 struct file *file = epi->ffd.file; in ep_remove()
784 ep_unregister_pollwait(ep, epi); in ep_remove()
788 list_del_rcu(&epi->fllink); in ep_remove()
791 rb_erase_cached(&epi->rbn, &ep->rbr); in ep_remove()
794 if (ep_is_linked(epi)) in ep_remove()
795 list_del_init(&epi->rdllink); in ep_remove()
798 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_remove()
806 call_rcu(&epi->rcu, epi_rcu_free); in ep_remove()
816 struct epitem *epi; in ep_free() local
836 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
838 ep_unregister_pollwait(ep, epi); in ep_free()
852 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
853 ep_remove(ep, epi); in ep_free()
885 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, in ep_item_poll() argument
891 pt->_key = epi->event.events; in ep_item_poll()
892 if (!is_file_epoll(epi->ffd.file)) in ep_item_poll()
893 return vfs_poll(epi->ffd.file, pt) & epi->event.events; in ep_item_poll()
895 ep = epi->ffd.file->private_data; in ep_item_poll()
896 poll_wait(epi->ffd.file, &ep->poll_wait, pt); in ep_item_poll()
899 return ep_scan_ready_list(epi->ffd.file->private_data, in ep_item_poll()
901 locked) & epi->event.events; in ep_item_poll()
907 struct epitem *epi, *tmp; in ep_read_events_proc() local
914 list_for_each_entry_safe(epi, tmp, head, rdllink) { in ep_read_events_proc()
915 if (ep_item_poll(epi, &pt, depth)) { in ep_read_events_proc()
923 __pm_relax(ep_wakeup_source(epi)); in ep_read_events_proc()
924 list_del_init(&epi->rdllink); in ep_read_events_proc()
955 struct epitem *epi = rb_entry(rbp, struct epitem, rbn); in ep_show_fdinfo() local
956 struct inode *inode = file_inode(epi->ffd.file); in ep_show_fdinfo()
960 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
961 (long long)epi->event.data, in ep_show_fdinfo()
962 (long long)epi->ffd.file->f_pos, in ep_show_fdinfo()
989 struct epitem *epi, *next; in eventpoll_release_file() local
1005 list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) { in eventpoll_release_file()
1006 ep = epi->ep; in eventpoll_release_file()
1008 ep_remove(ep, epi); in eventpoll_release_file()
1053 struct epitem *epi, *epir = NULL; in ep_find() local
1058 epi = rb_entry(rbp, struct epitem, rbn); in ep_find()
1059 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
1065 epir = epi; in ep_find()
1077 struct epitem *epi; in ep_find_tfd() local
1080 epi = rb_entry(rbp, struct epitem, rbn); in ep_find_tfd()
1081 if (epi->ffd.fd == tfd) { in ep_find_tfd()
1083 return epi; in ep_find_tfd()
1098 struct epitem *epi; in get_epoll_tfile_raw_ptr() local
1106 epi = ep_find_tfd(ep, tfd, toff); in get_epoll_tfile_raw_ptr()
1107 if (epi) in get_epoll_tfile_raw_ptr()
1108 file_raw = epi->ffd.file; in get_epoll_tfile_raw_ptr()
1176 static inline bool chain_epi_lockless(struct epitem *epi) in chain_epi_lockless() argument
1178 struct eventpoll *ep = epi->ep; in chain_epi_lockless()
1181 if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR) in chain_epi_lockless()
1185 epi->next = xchg(&ep->ovflist, epi); in chain_epi_lockless()
1211 struct epitem *epi = ep_item_from_wait(wait); in ep_poll_callback() local
1212 struct eventpoll *ep = epi->ep; in ep_poll_callback()
1219 ep_set_busy_poll_napi_id(epi); in ep_poll_callback()
1227 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1236 if (pollflags && !(pollflags & epi->event.events)) in ep_poll_callback()
1246 if (epi->next == EP_UNACTIVE_PTR && in ep_poll_callback()
1247 chain_epi_lockless(epi)) in ep_poll_callback()
1248 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1253 if (!ep_is_linked(epi) && in ep_poll_callback()
1254 list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) { in ep_poll_callback()
1255 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1263 if ((epi->event.events & EPOLLEXCLUSIVE) && in ep_poll_callback()
1267 if (epi->event.events & EPOLLIN) in ep_poll_callback()
1271 if (epi->event.events & EPOLLOUT) in ep_poll_callback()
1291 if (!(epi->event.events & EPOLLEXCLUSIVE)) in ep_poll_callback()
1320 struct epitem *epi = ep_item_from_epqueue(pt); in ep_ptable_queue_proc() local
1323 if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) { in ep_ptable_queue_proc()
1326 pwq->base = epi; in ep_ptable_queue_proc()
1327 if (epi->event.events & EPOLLEXCLUSIVE) in ep_ptable_queue_proc()
1331 list_add_tail(&pwq->llink, &epi->pwqlist); in ep_ptable_queue_proc()
1332 epi->nwait++; in ep_ptable_queue_proc()
1335 epi->nwait = -1; in ep_ptable_queue_proc()
1339 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1349 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1356 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1357 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); in ep_rbtree_insert()
1401 struct epitem *epi; in reverse_path_check_proc() local
1405 list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) { in reverse_path_check_proc()
1406 child_file = epi->ep->file; in reverse_path_check_proc()
1457 static int ep_create_wakeup_source(struct epitem *epi) in ep_create_wakeup_source() argument
1462 if (!epi->ep->ws) { in ep_create_wakeup_source()
1463 epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); in ep_create_wakeup_source()
1464 if (!epi->ep->ws) in ep_create_wakeup_source()
1468 name = epi->ffd.file->f_path.dentry->d_name.name; in ep_create_wakeup_source()
1473 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1479 static noinline void ep_destroy_wakeup_source(struct epitem *epi) in ep_destroy_wakeup_source() argument
1481 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_destroy_wakeup_source()
1483 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1503 struct epitem *epi; in ep_insert() local
1511 if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL))) in ep_insert()
1515 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1516 INIT_LIST_HEAD(&epi->fllink); in ep_insert()
1517 INIT_LIST_HEAD(&epi->pwqlist); in ep_insert()
1518 epi->ep = ep; in ep_insert()
1519 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1520 epi->event = *event; in ep_insert()
1521 epi->nwait = 0; in ep_insert()
1522 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1523 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1524 error = ep_create_wakeup_source(epi); in ep_insert()
1528 RCU_INIT_POINTER(epi->ws, NULL); in ep_insert()
1532 epq.epi = epi; in ep_insert()
1542 revents = ep_item_poll(epi, &epq.pt, 1); in ep_insert()
1550 if (epi->nwait < 0) in ep_insert()
1555 list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links); in ep_insert()
1562 ep_rbtree_insert(ep, epi); in ep_insert()
1573 ep_set_busy_poll_napi_id(epi); in ep_insert()
1576 if (revents && !ep_is_linked(epi)) { in ep_insert()
1577 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1578 ep_pm_stay_awake(epi); in ep_insert()
1599 list_del_rcu(&epi->fllink); in ep_insert()
1602 rb_erase_cached(&epi->rbn, &ep->rbr); in ep_insert()
1605 ep_unregister_pollwait(ep, epi); in ep_insert()
1614 if (ep_is_linked(epi)) in ep_insert()
1615 list_del_init(&epi->rdllink); in ep_insert()
1618 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_insert()
1621 kmem_cache_free(epi_cache, epi); in ep_insert()
1630 static int ep_modify(struct eventpoll *ep, struct epitem *epi, in ep_modify() argument
1645 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1646 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1647 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1648 if (!ep_has_wakeup_source(epi)) in ep_modify()
1649 ep_create_wakeup_source(epi); in ep_modify()
1650 } else if (ep_has_wakeup_source(epi)) { in ep_modify()
1651 ep_destroy_wakeup_source(epi); in ep_modify()
1680 if (ep_item_poll(epi, &pt, 1)) { in ep_modify()
1682 if (!ep_is_linked(epi)) { in ep_modify()
1683 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1684 ep_pm_stay_awake(epi); in ep_modify()
1707 struct epitem *epi, *tmp; in ep_send_events_proc() local
1722 list_for_each_entry_safe(epi, tmp, head, rdllink) { in ep_send_events_proc()
1735 ws = ep_wakeup_source(epi); in ep_send_events_proc()
1742 list_del_init(&epi->rdllink); in ep_send_events_proc()
1750 revents = ep_item_poll(epi, &pt, 1); in ep_send_events_proc()
1755 __put_user(epi->event.data, &uevent->data)) { in ep_send_events_proc()
1756 list_add(&epi->rdllink, head); in ep_send_events_proc()
1757 ep_pm_stay_awake(epi); in ep_send_events_proc()
1764 if (epi->event.events & EPOLLONESHOT) in ep_send_events_proc()
1765 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events_proc()
1766 else if (!(epi->event.events & EPOLLET)) { in ep_send_events_proc()
1778 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events_proc()
1779 ep_pm_stay_awake(epi); in ep_send_events_proc()
1966 struct epitem *epi; in ep_loop_check_proc() local
1972 epi = rb_entry(rbp, struct epitem, rbn); in ep_loop_check_proc()
1973 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
1974 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
1978 ep_loop_check_proc, epi->ffd.file, in ep_loop_check_proc()
1991 if (list_empty(&epi->ffd.file->f_tfile_llink)) in ep_loop_check_proc()
1992 list_add(&epi->ffd.file->f_tfile_llink, in ep_loop_check_proc()
2112 struct epitem *epi; in SYSCALL_DEFINE4() local
2212 epi = ep_find(ep, tf.file, fd); in SYSCALL_DEFINE4()
2217 if (!epi) { in SYSCALL_DEFINE4()
2226 if (epi) in SYSCALL_DEFINE4()
2227 error = ep_remove(ep, epi); in SYSCALL_DEFINE4()
2232 if (epi) { in SYSCALL_DEFINE4()
2233 if (!(epi->event.events & EPOLLEXCLUSIVE)) { in SYSCALL_DEFINE4()
2235 error = ep_modify(ep, epi, &epds); in SYSCALL_DEFINE4()