43 static inline bool lock_wqueue(struct watch_queue *wqueue)  in lock_wqueue()  argument
45 	spin_lock_bh(&wqueue->lock);  in lock_wqueue()
46 	if (unlikely(!wqueue->pipe)) {  in lock_wqueue()
47 		spin_unlock_bh(&wqueue->lock);  in lock_wqueue()
53 static inline void unlock_wqueue(struct watch_queue *wqueue)  in unlock_wqueue()  argument
55 	spin_unlock_bh(&wqueue->lock);  in unlock_wqueue()
61 	struct watch_queue *wqueue = (struct watch_queue *)buf->private;  in watch_queue_pipe_buf_release()  local
77 	set_bit(bit, wqueue->notes_bitmap);  in watch_queue_pipe_buf_release()
98 static bool post_one_notification(struct watch_queue *wqueue,  in post_one_notification()  argument
102 	struct pipe_inode_info *pipe = wqueue->pipe;  in post_one_notification()
116 	note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);  in post_one_notification()
117 	if (note >= wqueue->nr_notes)  in post_one_notification()
120 	page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];  in post_one_notification()
130 	buf->private = (unsigned long)wqueue;  in post_one_notification()
137 	if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {  in post_one_notification()
201 	struct watch_queue *wqueue;  in __post_watch_notification()  local
217 		wqueue = rcu_dereference(watch->queue);  in __post_watch_notification()
218 		wf = rcu_dereference(wqueue->filter);  in __post_watch_notification()
225 		if (lock_wqueue(wqueue)) {  in __post_watch_notification()
226 			post_one_notification(wqueue, n);  in __post_watch_notification()
227 			unlock_wqueue(wqueue);  in __post_watch_notification()
241 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_size()  local
247 	if (!wqueue)  in watch_queue_set_size()
249 	if (wqueue->notes)  in watch_queue_set_size()
290 	wqueue->notes = pages;  in watch_queue_set_size()
291 	wqueue->notes_bitmap = bitmap;  in watch_queue_set_size()
292 	wqueue->nr_pages = nr_pages;  in watch_queue_set_size()
293 	wqueue->nr_notes = nr_notes;  in watch_queue_set_size()
315 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_filter()  local
318 	if (!wqueue)  in watch_queue_set_filter()
375 	wfilter = rcu_replace_pointer(wqueue->filter, wfilter,  in watch_queue_set_filter()
389 	struct watch_queue *wqueue =  in __put_watch_queue()  local
394 	for (i = 0; i < wqueue->nr_pages; i++)  in __put_watch_queue()
395 		__free_page(wqueue->notes[i]);  in __put_watch_queue()
396 	kfree(wqueue->notes);  in __put_watch_queue()
397 	bitmap_free(wqueue->notes_bitmap);  in __put_watch_queue()
399 	wfilter = rcu_access_pointer(wqueue->filter);  in __put_watch_queue()
402 	kfree_rcu(wqueue, rcu);  in __put_watch_queue()
409 void put_watch_queue(struct watch_queue *wqueue)  in put_watch_queue()  argument
411 	kref_put(&wqueue->usage, __put_watch_queue);  in put_watch_queue()
447 void init_watch(struct watch *watch, struct watch_queue *wqueue)  in init_watch()  argument
452 	rcu_assign_pointer(watch->queue, wqueue);  in init_watch()
455 static int add_one_watch(struct watch *watch, struct watch_list *wlist, struct watch_queue *wqueue)  in add_one_watch()  argument
462 		if (wqueue == wq && watch->id == w->id)  in add_one_watch()
475 	kref_get(&wqueue->usage);  in add_one_watch()
477 	hlist_add_head(&watch->queue_node, &wqueue->watches);  in add_one_watch()
496 	struct watch_queue *wqueue;  in add_watch_to_object()  local
501 	wqueue = rcu_access_pointer(watch->queue);  in add_watch_to_object()
502 	if (lock_wqueue(wqueue)) {  in add_watch_to_object()
504 		ret = add_one_watch(watch, wlist, wqueue);  in add_watch_to_object()
506 		unlock_wqueue(wqueue);  in add_watch_to_object()
528 	struct watch_queue *wqueue;  in remove_watch_from_object()  local
559 	wqueue = rcu_dereference(watch->queue);  in remove_watch_from_object()
561 	if (lock_wqueue(wqueue)) {  in remove_watch_from_object()
562 		post_one_notification(wqueue, &n.watch);  in remove_watch_from_object()
569 		unlock_wqueue(wqueue);  in remove_watch_from_object()
595 void watch_queue_clear(struct watch_queue *wqueue)  in watch_queue_clear()  argument
602 	spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
608 	wqueue->pipe = NULL;  in watch_queue_clear()
610 	while (!hlist_empty(&wqueue->watches)) {  in watch_queue_clear()
611 		watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);  in watch_queue_clear()
614 		spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
652 		spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
655 	spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
666 	struct watch_queue *wqueue = ERR_PTR(-EINVAL);  in get_watch_queue()  local
673 			wqueue = pipe->watch_queue;  in get_watch_queue()
674 			kref_get(&wqueue->usage);  in get_watch_queue()
679 	return wqueue;  in get_watch_queue()
688 	struct watch_queue *wqueue;  in watch_queue_init()  local
690 	wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);  in watch_queue_init()
691 	if (!wqueue)  in watch_queue_init()
694 	wqueue->pipe = pipe;  in watch_queue_init()
695 	kref_init(&wqueue->usage);  in watch_queue_init()
696 	spin_lock_init(&wqueue->lock);  in watch_queue_init()
697 	INIT_HLIST_HEAD(&wqueue->watches);  in watch_queue_init()
699 	pipe->watch_queue = wqueue;  in watch_queue_init()