44 	struct mmu_notifier *mn;  in __mmu_notifier_release()  local
52 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)  in __mmu_notifier_release()
59 		if (mn->ops->release)  in __mmu_notifier_release()
60 			mn->ops->release(mn, mm);  in __mmu_notifier_release()
64 		mn = hlist_entry(mm->mmu_notifier_mm->list.first,  in __mmu_notifier_release()
73 		hlist_del_init_rcu(&mn->hlist);  in __mmu_notifier_release()
99 	struct mmu_notifier *mn;  in __mmu_notifier_clear_flush_young()  local
103 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_flush_young()
104 		if (mn->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
105 			young |= mn->ops->clear_flush_young(mn, mm, start, end);  in __mmu_notifier_clear_flush_young()
116 	struct mmu_notifier *mn;  in __mmu_notifier_clear_young()  local
120 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_young()
121 		if (mn->ops->clear_young)  in __mmu_notifier_clear_young()
122 			young |= mn->ops->clear_young(mn, mm, start, end);  in __mmu_notifier_clear_young()
132 	struct mmu_notifier *mn;  in __mmu_notifier_test_young()  local
136 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_test_young()
137 		if (mn->ops->test_young) {  in __mmu_notifier_test_young()
138 			young = mn->ops->test_young(mn, mm, address);  in __mmu_notifier_test_young()
151 	struct mmu_notifier *mn;  in __mmu_notifier_change_pte()  local
155 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_change_pte()
156 		if (mn->ops->change_pte)  in __mmu_notifier_change_pte()
157 			mn->ops->change_pte(mn, mm, address, pte);  in __mmu_notifier_change_pte()
164 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range_start()  local
169 	hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_start()
170 		if (mn->ops->invalidate_range_start) {  in __mmu_notifier_invalidate_range_start()
175 			_ret = mn->ops->invalidate_range_start(mn, range);  in __mmu_notifier_invalidate_range_start()
180 					mn->ops->invalidate_range_start, _ret,  in __mmu_notifier_invalidate_range_start()
196 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range_end()  local
201 	hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_end()
215 		if (!only_end && mn->ops->invalidate_range)  in __mmu_notifier_invalidate_range_end()
216 			mn->ops->invalidate_range(mn, range->mm,  in __mmu_notifier_invalidate_range_end()
219 		if (mn->ops->invalidate_range_end) {  in __mmu_notifier_invalidate_range_end()
222 			mn->ops->invalidate_range_end(mn, range);  in __mmu_notifier_invalidate_range_end()
234 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range()  local
238 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range()
239 		if (mn->ops->invalidate_range)  in __mmu_notifier_invalidate_range()
240 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range()
249 int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in __mmu_notifier_register()  argument
264 	mn->mm = mm;  in __mmu_notifier_register()
265 	mn->users = 1;  in __mmu_notifier_register()
301 	hlist_add_head_rcu(&mn->hlist, &mm->mmu_notifier_mm->list);  in __mmu_notifier_register()
333 int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_register()  argument
338 	ret = __mmu_notifier_register(mn, mm);  in mmu_notifier_register()
347 	struct mmu_notifier *mn;  in find_get_mmu_notifier()  local
350 	hlist_for_each_entry_rcu (mn, &mm->mmu_notifier_mm->list, hlist) {  in find_get_mmu_notifier()
351 		if (mn->ops != ops)  in find_get_mmu_notifier()
354 		if (likely(mn->users != UINT_MAX))  in find_get_mmu_notifier()
355 			mn->users++;  in find_get_mmu_notifier()
357 			mn = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
359 		return mn;  in find_get_mmu_notifier()
385 	struct mmu_notifier *mn;  in mmu_notifier_get_locked()  local
391 		mn = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
392 		if (mn)  in mmu_notifier_get_locked()
393 			return mn;  in mmu_notifier_get_locked()
396 	mn = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
397 	if (IS_ERR(mn))  in mmu_notifier_get_locked()
398 		return mn;  in mmu_notifier_get_locked()
399 	mn->ops = ops;  in mmu_notifier_get_locked()
400 	ret = __mmu_notifier_register(mn, mm);  in mmu_notifier_get_locked()
403 	return mn;  in mmu_notifier_get_locked()
405 	mn->ops->free_notifier(mn);  in mmu_notifier_get_locked()
428 void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_unregister()  argument
432 	if (!hlist_unhashed(&mn->hlist)) {  in mmu_notifier_unregister()
444 		if (mn->ops->release)  in mmu_notifier_unregister()
445 			mn->ops->release(mn, mm);  in mmu_notifier_unregister()
453 		hlist_del_init_rcu(&mn->hlist);  in mmu_notifier_unregister()
471 	struct mmu_notifier *mn = container_of(rcu, struct mmu_notifier, rcu);  in mmu_notifier_free_rcu()  local
472 	struct mm_struct *mm = mn->mm;  in mmu_notifier_free_rcu()
474 	mn->ops->free_notifier(mn);  in mmu_notifier_free_rcu()
501 void mmu_notifier_put(struct mmu_notifier *mn)  in mmu_notifier_put()  argument
503 	struct mm_struct *mm = mn->mm;  in mmu_notifier_put()
506 	if (WARN_ON(!mn->users) || --mn->users)  in mmu_notifier_put()
508 	hlist_del_init_rcu(&mn->hlist);  in mmu_notifier_put()
511 	call_srcu(&srcu, &mn->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()