57 void clear_page_mlock(struct page *page)  in clear_page_mlock()  argument
59 	if (!TestClearPageMlocked(page))  in clear_page_mlock()
62 	mod_zone_page_state(page_zone(page), NR_MLOCK,  in clear_page_mlock()
63 			    -hpage_nr_pages(page));  in clear_page_mlock()
65 	if (!isolate_lru_page(page)) {  in clear_page_mlock()
66 		putback_lru_page(page);  in clear_page_mlock()
71 		if (PageUnevictable(page))  in clear_page_mlock()
80 void mlock_vma_page(struct page *page)  in mlock_vma_page()  argument
83 	BUG_ON(!PageLocked(page));  in mlock_vma_page()
85 	if (!TestSetPageMlocked(page)) {  in mlock_vma_page()
86 		mod_zone_page_state(page_zone(page), NR_MLOCK,  in mlock_vma_page()
87 				    hpage_nr_pages(page));  in mlock_vma_page()
89 		if (!isolate_lru_page(page))  in mlock_vma_page()
90 			putback_lru_page(page);  in mlock_vma_page()
98 static bool __munlock_isolate_lru_page(struct page *page, bool getpage)  in __munlock_isolate_lru_page()  argument
100 	if (PageLRU(page)) {  in __munlock_isolate_lru_page()
103 		lruvec = mem_cgroup_page_lruvec(page, page_zone(page));  in __munlock_isolate_lru_page()
105 			get_page(page);  in __munlock_isolate_lru_page()
106 		ClearPageLRU(page);  in __munlock_isolate_lru_page()
107 		del_page_from_lru_list(page, lruvec, page_lru(page));  in __munlock_isolate_lru_page()
120 static void __munlock_isolated_page(struct page *page)  in __munlock_isolated_page()  argument
128 	if (page_mapcount(page) > 1)  in __munlock_isolated_page()
129 		ret = try_to_munlock(page);  in __munlock_isolated_page()
135 	putback_lru_page(page);  in __munlock_isolated_page()
147 static void __munlock_isolation_failed(struct page *page)  in __munlock_isolation_failed()  argument
149 	if (PageUnevictable(page))  in __munlock_isolation_failed()
173 unsigned int munlock_vma_page(struct page *page)  in munlock_vma_page()  argument
176 	struct zone *zone = page_zone(page);  in munlock_vma_page()
179 	BUG_ON(!PageLocked(page));  in munlock_vma_page()
188 	nr_pages = hpage_nr_pages(page);  in munlock_vma_page()
189 	if (!TestClearPageMlocked(page))  in munlock_vma_page()
194 	if (__munlock_isolate_lru_page(page, true)) {  in munlock_vma_page()
196 		__munlock_isolated_page(page);  in munlock_vma_page()
199 	__munlock_isolation_failed(page);  in munlock_vma_page()
288 static bool __putback_lru_fast_prepare(struct page *page, struct pagevec *pvec,  in __putback_lru_fast_prepare()  argument
291 	VM_BUG_ON_PAGE(PageLRU(page), page);  in __putback_lru_fast_prepare()
292 	VM_BUG_ON_PAGE(!PageLocked(page), page);  in __putback_lru_fast_prepare()
294 	if (page_mapcount(page) <= 1 && page_evictable(page)) {  in __putback_lru_fast_prepare()
295 		pagevec_add(pvec, page);  in __putback_lru_fast_prepare()
296 		if (TestClearPageUnevictable(page))  in __putback_lru_fast_prepare()
298 		unlock_page(page);  in __putback_lru_fast_prepare()
345 		struct page *page = pvec->pages[i];  in __munlock_pagevec()  local
347 		if (TestClearPageMlocked(page)) {  in __munlock_pagevec()
352 			if (__munlock_isolate_lru_page(page, false))  in __munlock_pagevec()
355 				__munlock_isolation_failed(page);  in __munlock_pagevec()
377 		struct page *page = pvec->pages[i];  in __munlock_pagevec()  local
379 		if (page) {  in __munlock_pagevec()
380 			lock_page(page);  in __munlock_pagevec()
381 			if (!__putback_lru_fast_prepare(page, &pvec_putback,  in __munlock_pagevec()
387 				get_page(page); /* for putback_lru_page() */  in __munlock_pagevec()
388 				__munlock_isolated_page(page);  in __munlock_pagevec()
389 				unlock_page(page);  in __munlock_pagevec()
390 				put_page(page); /* from follow_page_mask() */  in __munlock_pagevec()
437 		struct page *page = NULL;  in __munlock_pagevec_fill()  local
440 			page = vm_normal_page(vma, start, *pte);  in __munlock_pagevec_fill()
445 		if (!page || page_zone_id(page) != zoneid)  in __munlock_pagevec_fill()
448 		get_page(page);  in __munlock_pagevec_fill()
454 		if (pagevec_add(pvec, page) == 0)  in __munlock_pagevec_fill()
485 		struct page *page = NULL;  in munlock_vma_pages_range()  local
500 		page = follow_page_mask(vma, start, FOLL_GET | FOLL_DUMP,  in munlock_vma_pages_range()
503 		if (page && !IS_ERR(page)) {  in munlock_vma_pages_range()
504 			if (PageTransHuge(page)) {  in munlock_vma_pages_range()
505 				lock_page(page);  in munlock_vma_pages_range()
512 				page_mask = munlock_vma_page(page);  in munlock_vma_pages_range()
513 				unlock_page(page);  in munlock_vma_pages_range()
514 				put_page(page); /* follow_page_mask() */  in munlock_vma_pages_range()
521 				pagevec_add(&pvec, page);  in munlock_vma_pages_range()
522 				zone = page_zone(page);  in munlock_vma_pages_range()
523 				zoneid = page_zone_id(page);  in munlock_vma_pages_range()