Lines Matching refs:epi
241 struct epitem *epi; member
349 return container_of(p, struct ep_pqueue, pt)->epi; in ep_item_from_epqueue()
529 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
531 struct list_head *lsthead = &epi->pwqlist; in ep_unregister_pollwait()
544 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) in ep_wakeup_source() argument
546 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
550 static inline void ep_pm_stay_awake(struct epitem *epi) in ep_pm_stay_awake() argument
552 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_pm_stay_awake()
558 static inline bool ep_has_wakeup_source(struct epitem *epi) in ep_has_wakeup_source() argument
560 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
564 static inline void ep_pm_stay_awake_rcu(struct epitem *epi) in ep_pm_stay_awake_rcu() argument
569 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
595 struct epitem *epi, *nepi; in ep_scan_ready_list() local
628 for (nepi = ep->ovflist; (epi = nepi) != NULL; in ep_scan_ready_list()
629 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_scan_ready_list()
636 if (!ep_is_linked(&epi->rdllink)) { in ep_scan_ready_list()
637 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_scan_ready_list()
638 ep_pm_stay_awake(epi); in ep_scan_ready_list()
679 static int ep_remove(struct eventpoll *ep, struct epitem *epi) in ep_remove() argument
682 struct file *file = epi->ffd.file; in ep_remove()
692 ep_unregister_pollwait(ep, epi); in ep_remove()
696 if (ep_is_linked(&epi->fllink)) in ep_remove()
697 list_del_init(&epi->fllink); in ep_remove()
700 rb_erase(&epi->rbn, &ep->rbr); in ep_remove()
703 if (ep_is_linked(&epi->rdllink)) in ep_remove()
704 list_del_init(&epi->rdllink); in ep_remove()
707 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_remove()
710 kmem_cache_free(epi_cache, epi); in ep_remove()
720 struct epitem *epi; in ep_free() local
740 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
742 ep_unregister_pollwait(ep, epi); in ep_free()
755 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
756 ep_remove(ep, epi); in ep_free()
777 static inline unsigned int ep_item_poll(struct epitem *epi, poll_table *pt) in ep_item_poll() argument
779 pt->_key = epi->event.events; in ep_item_poll()
781 return epi->ffd.file->f_op->poll(epi->ffd.file, pt) & epi->event.events; in ep_item_poll()
787 struct epitem *epi, *tmp; in ep_read_events_proc() local
792 list_for_each_entry_safe(epi, tmp, head, rdllink) { in ep_read_events_proc()
793 if (ep_item_poll(epi, &pt)) in ep_read_events_proc()
801 __pm_relax(ep_wakeup_source(epi)); in ep_read_events_proc()
802 list_del_init(&epi->rdllink); in ep_read_events_proc()
843 struct epitem *epi = rb_entry(rbp, struct epitem, rbn); in ep_show_fdinfo() local
846 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
847 (long long)epi->event.data); in ep_show_fdinfo()
876 struct epitem *epi; in eventpoll_release_file() local
894 epi = list_first_entry(lsthead, struct epitem, fllink); in eventpoll_release_file()
896 ep = epi->ep; in eventpoll_release_file()
897 list_del_init(&epi->fllink); in eventpoll_release_file()
899 ep_remove(ep, epi); in eventpoll_release_file()
945 struct epitem *epi, *epir = NULL; in ep_find() local
950 epi = rb_entry(rbp, struct epitem, rbn); in ep_find()
951 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
957 epir = epi; in ep_find()
974 struct epitem *epi = ep_item_from_wait(wait); in ep_poll_callback() local
975 struct eventpoll *ep = epi->ep; in ep_poll_callback()
996 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1005 if (key && !((unsigned long) key & epi->event.events)) in ep_poll_callback()
1015 if (epi->next == EP_UNACTIVE_PTR) { in ep_poll_callback()
1016 epi->next = ep->ovflist; in ep_poll_callback()
1017 ep->ovflist = epi; in ep_poll_callback()
1018 if (epi->ws) { in ep_poll_callback()
1031 if (!ep_is_linked(&epi->rdllink)) { in ep_poll_callback()
1032 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_poll_callback()
1033 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1062 struct epitem *epi = ep_item_from_epqueue(pt); in ep_ptable_queue_proc() local
1065 if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) { in ep_ptable_queue_proc()
1068 pwq->base = epi; in ep_ptable_queue_proc()
1070 list_add_tail(&pwq->llink, &epi->pwqlist); in ep_ptable_queue_proc()
1071 epi->nwait++; in ep_ptable_queue_proc()
1074 epi->nwait = -1; in ep_ptable_queue_proc()
1078 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1087 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1093 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1094 rb_insert_color(&epi->rbn, &ep->rbr); in ep_rbtree_insert()
1138 struct epitem *epi; in reverse_path_check_proc() local
1140 list_for_each_entry(epi, &file->f_ep_links, fllink) { in reverse_path_check_proc()
1141 child_file = epi->ep->file; in reverse_path_check_proc()
1192 static int ep_create_wakeup_source(struct epitem *epi) in ep_create_wakeup_source() argument
1197 if (!epi->ep->ws) { in ep_create_wakeup_source()
1198 epi->ep->ws = wakeup_source_register("eventpoll"); in ep_create_wakeup_source()
1199 if (!epi->ep->ws) in ep_create_wakeup_source()
1203 name = epi->ffd.file->f_path.dentry->d_name.name; in ep_create_wakeup_source()
1208 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1214 static noinline void ep_destroy_wakeup_source(struct epitem *epi) in ep_destroy_wakeup_source() argument
1216 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_destroy_wakeup_source()
1218 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1238 struct epitem *epi; in ep_insert() local
1244 if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL))) in ep_insert()
1248 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1249 INIT_LIST_HEAD(&epi->fllink); in ep_insert()
1250 INIT_LIST_HEAD(&epi->pwqlist); in ep_insert()
1251 epi->ep = ep; in ep_insert()
1252 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1253 epi->event = *event; in ep_insert()
1254 epi->nwait = 0; in ep_insert()
1255 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1256 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1257 error = ep_create_wakeup_source(epi); in ep_insert()
1261 RCU_INIT_POINTER(epi->ws, NULL); in ep_insert()
1265 epq.epi = epi; in ep_insert()
1275 revents = ep_item_poll(epi, &epq.pt); in ep_insert()
1283 if (epi->nwait < 0) in ep_insert()
1288 list_add_tail(&epi->fllink, &tfile->f_ep_links); in ep_insert()
1295 ep_rbtree_insert(ep, epi); in ep_insert()
1306 if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) { in ep_insert()
1307 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1308 ep_pm_stay_awake(epi); in ep_insert()
1329 if (ep_is_linked(&epi->fllink)) in ep_insert()
1330 list_del_init(&epi->fllink); in ep_insert()
1333 rb_erase(&epi->rbn, &ep->rbr); in ep_insert()
1336 ep_unregister_pollwait(ep, epi); in ep_insert()
1345 if (ep_is_linked(&epi->rdllink)) in ep_insert()
1346 list_del_init(&epi->rdllink); in ep_insert()
1349 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_insert()
1352 kmem_cache_free(epi_cache, epi); in ep_insert()
1361 static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event) in ep_modify() argument
1374 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1375 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1376 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1377 if (!ep_has_wakeup_source(epi)) in ep_modify()
1378 ep_create_wakeup_source(epi); in ep_modify()
1379 } else if (ep_has_wakeup_source(epi)) { in ep_modify()
1380 ep_destroy_wakeup_source(epi); in ep_modify()
1407 revents = ep_item_poll(epi, &pt); in ep_modify()
1415 if (!ep_is_linked(&epi->rdllink)) { in ep_modify()
1416 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1417 ep_pm_stay_awake(epi); in ep_modify()
1441 struct epitem *epi; in ep_send_events_proc() local
1455 epi = list_first_entry(head, struct epitem, rdllink); in ep_send_events_proc()
1466 ws = ep_wakeup_source(epi); in ep_send_events_proc()
1473 list_del_init(&epi->rdllink); in ep_send_events_proc()
1475 revents = ep_item_poll(epi, &pt); in ep_send_events_proc()
1485 __put_user(epi->event.data, &uevent->data)) { in ep_send_events_proc()
1486 list_add(&epi->rdllink, head); in ep_send_events_proc()
1487 ep_pm_stay_awake(epi); in ep_send_events_proc()
1492 if (epi->event.events & EPOLLONESHOT) in ep_send_events_proc()
1493 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events_proc()
1494 else if (!(epi->event.events & EPOLLET)) { in ep_send_events_proc()
1506 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events_proc()
1507 ep_pm_stay_awake(epi); in ep_send_events_proc()
1656 struct epitem *epi; in ep_loop_check_proc() local
1662 epi = rb_entry(rbp, struct epitem, rbn); in ep_loop_check_proc()
1663 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
1664 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
1668 ep_loop_check_proc, epi->ffd.file, in ep_loop_check_proc()
1681 if (list_empty(&epi->ffd.file->f_tfile_llink)) in ep_loop_check_proc()
1682 list_add(&epi->ffd.file->f_tfile_llink, in ep_loop_check_proc()
1797 struct epitem *epi; in SYSCALL_DEFINE4() local
1874 epi = ep_find(ep, tfile, fd); in SYSCALL_DEFINE4()
1879 if (!epi) { in SYSCALL_DEFINE4()
1887 if (epi) in SYSCALL_DEFINE4()
1888 error = ep_remove(ep, epi); in SYSCALL_DEFINE4()
1893 if (epi) { in SYSCALL_DEFINE4()
1895 error = ep_modify(ep, epi, &epds); in SYSCALL_DEFINE4()