56 void __mmu_notifier_release(struct mm_struct *mm)  in __mmu_notifier_release()  argument
66 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)  in __mmu_notifier_release()
74 			mn->ops->release(mn, mm);  in __mmu_notifier_release()
76 	spin_lock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_release()
77 	while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {  in __mmu_notifier_release()
78 		mn = hlist_entry(mm->mmu_notifier_mm->list.first,  in __mmu_notifier_release()
89 	spin_unlock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_release()
109 int __mmu_notifier_clear_flush_young(struct mm_struct *mm,  in __mmu_notifier_clear_flush_young()  argument
117 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_flush_young()
119 			young |= mn->ops->clear_flush_young(mn, mm, start, end);  in __mmu_notifier_clear_flush_young()
126 int __mmu_notifier_test_young(struct mm_struct *mm,  in __mmu_notifier_test_young()  argument
133 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_test_young()
135 			young = mn->ops->test_young(mn, mm, address);  in __mmu_notifier_test_young()
145 void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,  in __mmu_notifier_change_pte()  argument
152 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_change_pte()
154 			mn->ops->change_pte(mn, mm, address, pte);  in __mmu_notifier_change_pte()
159 void __mmu_notifier_invalidate_page(struct mm_struct *mm,  in __mmu_notifier_invalidate_page()  argument
166 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_page()
168 			mn->ops->invalidate_page(mn, mm, address);  in __mmu_notifier_invalidate_page()
173 void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,  in __mmu_notifier_invalidate_range_start()  argument
180 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_start()
182 			mn->ops->invalidate_range_start(mn, mm, start, end);  in __mmu_notifier_invalidate_range_start()
188 void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,  in __mmu_notifier_invalidate_range_end()  argument
195 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_end()
197 			mn->ops->invalidate_range_end(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
204 				    struct mm_struct *mm,  in do_mmu_notifier_register()  argument
210 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in do_mmu_notifier_register()
224 		down_write(&mm->mmap_sem);  in do_mmu_notifier_register()
225 	ret = mm_take_all_locks(mm);  in do_mmu_notifier_register()
229 	if (!mm_has_notifiers(mm)) {  in do_mmu_notifier_register()
233 		mm->mmu_notifier_mm = mmu_notifier_mm;  in do_mmu_notifier_register()
236 	atomic_inc(&mm->mm_count);  in do_mmu_notifier_register()
246 	spin_lock(&mm->mmu_notifier_mm->lock);  in do_mmu_notifier_register()
247 	hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);  in do_mmu_notifier_register()
248 	spin_unlock(&mm->mmu_notifier_mm->lock);  in do_mmu_notifier_register()
250 	mm_drop_all_locks(mm);  in do_mmu_notifier_register()
253 		up_write(&mm->mmap_sem);  in do_mmu_notifier_register()
256 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in do_mmu_notifier_register()
273 int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_register()  argument
275 	return do_mmu_notifier_register(mn, mm, 1);  in mmu_notifier_register()
283 int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in __mmu_notifier_register()  argument
285 	return do_mmu_notifier_register(mn, mm, 0);  in __mmu_notifier_register()
290 void __mmu_notifier_mm_destroy(struct mm_struct *mm)  in __mmu_notifier_mm_destroy()  argument
292 	BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));  in __mmu_notifier_mm_destroy()
293 	kfree(mm->mmu_notifier_mm);  in __mmu_notifier_mm_destroy()
294 	mm->mmu_notifier_mm = LIST_POISON1; /* debug */  in __mmu_notifier_mm_destroy()
307 void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_unregister()  argument
309 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
324 			mn->ops->release(mn, mm);  in mmu_notifier_unregister()
327 		spin_lock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister()
333 		spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister()
342 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
344 	mmdrop(mm);  in mmu_notifier_unregister()
352 					struct mm_struct *mm)  in mmu_notifier_unregister_no_release()  argument
354 	spin_lock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister_no_release()
360 	spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister_no_release()
362 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister_no_release()
363 	mmdrop(mm);  in mmu_notifier_unregister_no_release()