Lines Matching refs:page
51 static void __page_cache_release(struct page *page) in __page_cache_release() argument
53 if (PageLRU(page)) { in __page_cache_release()
54 struct zone *zone = page_zone(page); in __page_cache_release()
59 lruvec = mem_cgroup_page_lruvec(page, zone); in __page_cache_release()
60 VM_BUG_ON_PAGE(!PageLRU(page), page); in __page_cache_release()
61 __ClearPageLRU(page); in __page_cache_release()
62 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in __page_cache_release()
65 mem_cgroup_uncharge(page); in __page_cache_release()
68 static void __put_single_page(struct page *page) in __put_single_page() argument
70 __page_cache_release(page); in __put_single_page()
71 free_hot_cold_page(page, false); in __put_single_page()
74 static void __put_compound_page(struct page *page) in __put_compound_page() argument
78 __page_cache_release(page); in __put_compound_page()
79 dtor = get_compound_page_dtor(page); in __put_compound_page()
80 (*dtor)(page); in __put_compound_page()
110 void put_unrefcounted_compound_page(struct page *page_head, struct page *page) in put_unrefcounted_compound_page() argument
120 if (likely(PageTail(page))) { in put_unrefcounted_compound_page()
126 VM_BUG_ON_PAGE(page_mapcount(page) != 0, page); in put_unrefcounted_compound_page()
153 if (put_page_testzero(page)) in put_unrefcounted_compound_page()
154 __put_single_page(page); in put_unrefcounted_compound_page()
158 void put_refcounted_compound_page(struct page *page_head, struct page *page) in put_refcounted_compound_page() argument
160 if (likely(page != page_head && get_page_unless_zero(page_head))) { in put_refcounted_compound_page()
170 if (unlikely(!PageTail(page))) { in put_refcounted_compound_page()
192 if (put_page_testzero(page)) in put_refcounted_compound_page()
193 __put_single_page(page); in put_refcounted_compound_page()
196 VM_BUG_ON_PAGE(page_head != page->first_page, page); in put_refcounted_compound_page()
206 VM_BUG_ON_PAGE(page_mapcount(page) <= 0, page); in put_refcounted_compound_page()
207 atomic_dec(&page->_mapcount); in put_refcounted_compound_page()
209 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page); in put_refcounted_compound_page()
220 VM_BUG_ON_PAGE(PageTail(page), page); in put_refcounted_compound_page()
225 static void put_compound_page(struct page *page) in put_compound_page() argument
227 struct page *page_head; in put_compound_page()
234 if (likely(!PageTail(page))) { in put_compound_page()
235 if (put_page_testzero(page)) { in put_compound_page()
240 if (PageHead(page)) in put_compound_page()
241 __put_compound_page(page); in put_compound_page()
243 __put_single_page(page); in put_compound_page()
257 page_head = compound_head_by_tail(page); in put_compound_page()
259 put_unrefcounted_compound_page(page_head, page); in put_compound_page()
261 put_refcounted_compound_page(page_head, page); in put_compound_page()
264 void put_page(struct page *page) in put_page() argument
266 if (unlikely(PageCompound(page))) in put_page()
267 put_compound_page(page); in put_page()
268 else if (put_page_testzero(page)) in put_page()
269 __put_single_page(page); in put_page()
277 bool __get_page_tail(struct page *page) in __get_page_tail() argument
289 struct page *page_head = compound_head(page); in __get_page_tail()
294 if (likely(PageTail(page))) { in __get_page_tail()
301 __get_page_tail_foll(page, true); in __get_page_tail()
318 if (likely(page != page_head && get_page_unless_zero(page_head))) { in __get_page_tail()
327 if (likely(PageTail(page))) { in __get_page_tail()
328 __get_page_tail_foll(page, false); in __get_page_tail()
349 struct page *victim; in put_pages_list()
351 victim = list_entry(pages->prev, struct page, lru); in put_pages_list()
372 struct page **pages) in get_kernel_pages()
399 int get_kernel_page(unsigned long start, int write, struct page **pages) in get_kernel_page()
411 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), in pagevec_lru_move_fn() argument
420 struct page *page = pvec->pages[i]; in pagevec_lru_move_fn() local
421 struct zone *pagezone = page_zone(page); in pagevec_lru_move_fn()
430 lruvec = mem_cgroup_page_lruvec(page, zone); in pagevec_lru_move_fn()
431 (*move_fn)(page, lruvec, arg); in pagevec_lru_move_fn()
439 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, in pagevec_move_tail_fn() argument
444 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in pagevec_move_tail_fn()
445 enum lru_list lru = page_lru_base_type(page); in pagevec_move_tail_fn()
446 list_move_tail(&page->lru, &lruvec->lists[lru]); in pagevec_move_tail_fn()
468 void rotate_reclaimable_page(struct page *page) in rotate_reclaimable_page() argument
470 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) && in rotate_reclaimable_page()
471 !PageUnevictable(page) && PageLRU(page)) { in rotate_reclaimable_page()
475 page_cache_get(page); in rotate_reclaimable_page()
478 if (!pagevec_add(pvec, page) || PageCompound(page)) in rotate_reclaimable_page()
494 static void __activate_page(struct page *page, struct lruvec *lruvec, in __activate_page() argument
497 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in __activate_page()
498 int file = page_is_file_cache(page); in __activate_page()
499 int lru = page_lru_base_type(page); in __activate_page()
501 del_page_from_lru_list(page, lruvec, lru); in __activate_page()
502 SetPageActive(page); in __activate_page()
504 add_page_to_lru_list(page, lruvec, lru); in __activate_page()
505 trace_mm_lru_activate(page); in __activate_page()
528 void activate_page(struct page *page) in activate_page() argument
530 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in activate_page()
533 page_cache_get(page); in activate_page()
534 if (!pagevec_add(pvec, page) || PageCompound(page)) in activate_page()
550 void activate_page(struct page *page) in activate_page() argument
552 struct zone *zone = page_zone(page); in activate_page()
555 __activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL); in activate_page()
560 static void __lru_cache_activate_page(struct page *page) in __lru_cache_activate_page() argument
576 struct page *pagevec_page = pvec->pages[i]; in __lru_cache_activate_page()
578 if (pagevec_page == page) { in __lru_cache_activate_page()
579 SetPageActive(page); in __lru_cache_activate_page()
597 void mark_page_accessed(struct page *page) in mark_page_accessed() argument
599 if (!PageActive(page) && !PageUnevictable(page) && in mark_page_accessed()
600 PageReferenced(page)) { in mark_page_accessed()
608 if (PageLRU(page)) in mark_page_accessed()
609 activate_page(page); in mark_page_accessed()
611 __lru_cache_activate_page(page); in mark_page_accessed()
612 ClearPageReferenced(page); in mark_page_accessed()
613 if (page_is_file_cache(page)) in mark_page_accessed()
614 workingset_activation(page); in mark_page_accessed()
615 } else if (!PageReferenced(page)) { in mark_page_accessed()
616 SetPageReferenced(page); in mark_page_accessed()
621 static void __lru_cache_add(struct page *page) in __lru_cache_add() argument
625 page_cache_get(page); in __lru_cache_add()
626 if (!pagevec_add(pvec, page) || PageCompound(page)) in __lru_cache_add()
635 void lru_cache_add_anon(struct page *page) in lru_cache_add_anon() argument
637 if (PageActive(page)) in lru_cache_add_anon()
638 ClearPageActive(page); in lru_cache_add_anon()
639 __lru_cache_add(page); in lru_cache_add_anon()
642 void lru_cache_add_file(struct page *page) in lru_cache_add_file() argument
644 if (PageActive(page)) in lru_cache_add_file()
645 ClearPageActive(page); in lru_cache_add_file()
646 __lru_cache_add(page); in lru_cache_add_file()
659 void lru_cache_add(struct page *page) in lru_cache_add() argument
661 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); in lru_cache_add()
662 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add()
663 __lru_cache_add(page); in lru_cache_add()
676 void add_page_to_unevictable_list(struct page *page) in add_page_to_unevictable_list() argument
678 struct zone *zone = page_zone(page); in add_page_to_unevictable_list()
682 lruvec = mem_cgroup_page_lruvec(page, zone); in add_page_to_unevictable_list()
683 ClearPageActive(page); in add_page_to_unevictable_list()
684 SetPageUnevictable(page); in add_page_to_unevictable_list()
685 SetPageLRU(page); in add_page_to_unevictable_list()
686 add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE); in add_page_to_unevictable_list()
700 void lru_cache_add_active_or_unevictable(struct page *page, in lru_cache_add_active_or_unevictable() argument
703 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add_active_or_unevictable()
706 SetPageActive(page); in lru_cache_add_active_or_unevictable()
707 lru_cache_add(page); in lru_cache_add_active_or_unevictable()
711 if (!TestSetPageMlocked(page)) { in lru_cache_add_active_or_unevictable()
717 __mod_zone_page_state(page_zone(page), NR_MLOCK, in lru_cache_add_active_or_unevictable()
718 hpage_nr_pages(page)); in lru_cache_add_active_or_unevictable()
721 add_page_to_unevictable_list(page); in lru_cache_add_active_or_unevictable()
745 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, in lru_deactivate_file_fn() argument
751 if (!PageLRU(page)) in lru_deactivate_file_fn()
754 if (PageUnevictable(page)) in lru_deactivate_file_fn()
758 if (page_mapped(page)) in lru_deactivate_file_fn()
761 active = PageActive(page); in lru_deactivate_file_fn()
762 file = page_is_file_cache(page); in lru_deactivate_file_fn()
763 lru = page_lru_base_type(page); in lru_deactivate_file_fn()
765 del_page_from_lru_list(page, lruvec, lru + active); in lru_deactivate_file_fn()
766 ClearPageActive(page); in lru_deactivate_file_fn()
767 ClearPageReferenced(page); in lru_deactivate_file_fn()
768 add_page_to_lru_list(page, lruvec, lru); in lru_deactivate_file_fn()
770 if (PageWriteback(page) || PageDirty(page)) { in lru_deactivate_file_fn()
776 SetPageReclaim(page); in lru_deactivate_file_fn()
782 list_move_tail(&page->lru, &lruvec->lists[lru]); in lru_deactivate_file_fn()
828 void deactivate_file_page(struct page *page) in deactivate_file_page() argument
834 if (PageUnevictable(page)) in deactivate_file_page()
837 if (likely(get_page_unless_zero(page))) { in deactivate_file_page()
840 if (!pagevec_add(pvec, page) || PageCompound(page)) in deactivate_file_page()
898 void release_pages(struct page **pages, int nr, bool cold) in release_pages()
908 struct page *page = pages[i]; in release_pages() local
910 if (unlikely(PageCompound(page))) { in release_pages()
915 put_compound_page(page); in release_pages()
929 if (!put_page_testzero(page)) in release_pages()
932 if (PageLRU(page)) { in release_pages()
933 struct zone *pagezone = page_zone(page); in release_pages()
944 lruvec = mem_cgroup_page_lruvec(page, zone); in release_pages()
945 VM_BUG_ON_PAGE(!PageLRU(page), page); in release_pages()
946 __ClearPageLRU(page); in release_pages()
947 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in release_pages()
951 __ClearPageActive(page); in release_pages()
953 list_add(&page->lru, &pages_to_free); in release_pages()
983 void lru_add_page_tail(struct page *page, struct page *page_tail, in lru_add_page_tail() argument
988 VM_BUG_ON_PAGE(!PageHead(page), page); in lru_add_page_tail()
989 VM_BUG_ON_PAGE(PageCompound(page_tail), page); in lru_add_page_tail()
990 VM_BUG_ON_PAGE(PageLRU(page_tail), page); in lru_add_page_tail()
997 if (likely(PageLRU(page))) in lru_add_page_tail()
998 list_add_tail(&page_tail->lru, &page->lru); in lru_add_page_tail()
1017 if (!PageUnevictable(page)) in lru_add_page_tail()
1022 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, in __pagevec_lru_add_fn() argument
1025 int file = page_is_file_cache(page); in __pagevec_lru_add_fn()
1026 int active = PageActive(page); in __pagevec_lru_add_fn()
1027 enum lru_list lru = page_lru(page); in __pagevec_lru_add_fn()
1029 VM_BUG_ON_PAGE(PageLRU(page), page); in __pagevec_lru_add_fn()
1031 SetPageLRU(page); in __pagevec_lru_add_fn()
1032 add_page_to_lru_list(page, lruvec, lru); in __pagevec_lru_add_fn()
1034 trace_mm_lru_insertion(page, lru); in __pagevec_lru_add_fn()
1091 struct page *page = pvec->pages[i]; in pagevec_remove_exceptionals() local
1092 if (!radix_tree_exceptional_entry(page)) in pagevec_remove_exceptionals()
1093 pvec->pages[j++] = page; in pagevec_remove_exceptionals()