Lines Matching refs:page
79 static void __page_cache_release(struct page *page) in __page_cache_release() argument
81 if (PageLRU(page)) { in __page_cache_release()
82 pg_data_t *pgdat = page_pgdat(page); in __page_cache_release()
87 lruvec = mem_cgroup_page_lruvec(page, pgdat); in __page_cache_release()
88 VM_BUG_ON_PAGE(!PageLRU(page), page); in __page_cache_release()
89 __ClearPageLRU(page); in __page_cache_release()
90 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in __page_cache_release()
93 __ClearPageWaiters(page); in __page_cache_release()
96 static void __put_single_page(struct page *page) in __put_single_page() argument
98 __page_cache_release(page); in __put_single_page()
99 mem_cgroup_uncharge(page); in __put_single_page()
100 free_unref_page(page); in __put_single_page()
103 static void __put_compound_page(struct page *page) in __put_compound_page() argument
111 if (!PageHuge(page)) in __put_compound_page()
112 __page_cache_release(page); in __put_compound_page()
113 destroy_compound_page(page); in __put_compound_page()
116 void __put_page(struct page *page) in __put_page() argument
118 if (is_zone_device_page(page)) { in __put_page()
119 put_dev_pagemap(page->pgmap); in __put_page()
128 if (unlikely(PageCompound(page))) in __put_page()
129 __put_compound_page(page); in __put_page()
131 __put_single_page(page); in __put_page()
145 struct page *victim; in put_pages_list()
168 struct page **pages) in get_kernel_pages()
195 int get_kernel_page(unsigned long start, int write, struct page **pages) in get_kernel_page()
207 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), in pagevec_lru_move_fn() argument
216 struct page *page = pvec->pages[i]; in pagevec_lru_move_fn() local
217 struct pglist_data *pagepgdat = page_pgdat(page); in pagevec_lru_move_fn()
226 lruvec = mem_cgroup_page_lruvec(page, pgdat); in pagevec_lru_move_fn()
227 (*move_fn)(page, lruvec, arg); in pagevec_lru_move_fn()
235 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, in pagevec_move_tail_fn() argument
240 if (PageLRU(page) && !PageUnevictable(page)) { in pagevec_move_tail_fn()
241 del_page_from_lru_list(page, lruvec, page_lru(page)); in pagevec_move_tail_fn()
242 ClearPageActive(page); in pagevec_move_tail_fn()
243 add_page_to_lru_list_tail(page, lruvec, page_lru(page)); in pagevec_move_tail_fn()
244 (*pgmoved) += thp_nr_pages(page); in pagevec_move_tail_fn()
265 void rotate_reclaimable_page(struct page *page) in rotate_reclaimable_page() argument
267 if (!PageLocked(page) && !PageDirty(page) && in rotate_reclaimable_page()
268 !PageUnevictable(page) && PageLRU(page)) { in rotate_reclaimable_page()
272 get_page(page); in rotate_reclaimable_page()
275 if (!pagevec_add(pvec, page) || PageCompound(page)) in rotate_reclaimable_page()
312 void lru_note_cost_page(struct page *page) in lru_note_cost_page() argument
315 if (page_is_file_lru(page)) { in lru_note_cost_page()
316 lru_note_cost(&(page_pgdat(page)->__lruvec), 1, thp_nr_pages(page)); in lru_note_cost_page()
320 lru_note_cost(mem_cgroup_page_lruvec(page, page_pgdat(page)), in lru_note_cost_page()
321 page_is_file_lru(page), thp_nr_pages(page)); in lru_note_cost_page()
324 static void __activate_page(struct page *page, struct lruvec *lruvec, in __activate_page() argument
327 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in __activate_page()
328 int lru = page_lru_base_type(page); in __activate_page()
329 int nr_pages = thp_nr_pages(page); in __activate_page()
331 del_page_from_lru_list(page, lruvec, lru); in __activate_page()
332 SetPageActive(page); in __activate_page()
334 add_page_to_lru_list(page, lruvec, lru); in __activate_page()
335 trace_mm_lru_activate(page); in __activate_page()
357 static void activate_page(struct page *page) in activate_page() argument
359 page = compound_head(page); in activate_page()
360 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { in activate_page()
365 get_page(page); in activate_page()
366 if (!pagevec_add(pvec, page) || PageCompound(page)) in activate_page()
377 static void activate_page(struct page *page) in activate_page() argument
379 pg_data_t *pgdat = page_pgdat(page); in activate_page()
381 page = compound_head(page); in activate_page()
383 __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); in activate_page()
388 static void __lru_cache_activate_page(struct page *page) in __lru_cache_activate_page() argument
407 struct page *pagevec_page = pvec->pages[i]; in __lru_cache_activate_page()
409 if (pagevec_page == page) { in __lru_cache_activate_page()
410 SetPageActive(page); in __lru_cache_activate_page()
428 void mark_page_accessed(struct page *page) in mark_page_accessed() argument
430 page = compound_head(page); in mark_page_accessed()
432 if (!PageReferenced(page)) { in mark_page_accessed()
433 SetPageReferenced(page); in mark_page_accessed()
434 } else if (PageUnevictable(page)) { in mark_page_accessed()
440 } else if (!PageActive(page)) { in mark_page_accessed()
447 if (PageLRU(page)) in mark_page_accessed()
448 activate_page(page); in mark_page_accessed()
450 __lru_cache_activate_page(page); in mark_page_accessed()
451 ClearPageReferenced(page); in mark_page_accessed()
452 workingset_activation(page); in mark_page_accessed()
454 if (page_is_idle(page)) in mark_page_accessed()
455 clear_page_idle(page); in mark_page_accessed()
468 void lru_cache_add(struct page *page) in lru_cache_add() argument
472 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); in lru_cache_add()
473 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add()
475 get_page(page); in lru_cache_add()
478 if (!pagevec_add(pvec, page) || PageCompound(page)) in lru_cache_add()
492 void lru_cache_add_inactive_or_unevictable(struct page *page, in lru_cache_add_inactive_or_unevictable() argument
497 VM_BUG_ON_PAGE(PageLRU(page), page); in lru_cache_add_inactive_or_unevictable()
500 if (unlikely(unevictable) && !TestSetPageMlocked(page)) { in lru_cache_add_inactive_or_unevictable()
501 int nr_pages = thp_nr_pages(page); in lru_cache_add_inactive_or_unevictable()
507 __mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); in lru_cache_add_inactive_or_unevictable()
510 lru_cache_add(page); in lru_cache_add_inactive_or_unevictable()
534 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, in lru_deactivate_file_fn() argument
539 int nr_pages = thp_nr_pages(page); in lru_deactivate_file_fn()
541 if (!PageLRU(page)) in lru_deactivate_file_fn()
544 if (PageUnevictable(page)) in lru_deactivate_file_fn()
548 if (page_mapped(page)) in lru_deactivate_file_fn()
551 active = PageActive(page); in lru_deactivate_file_fn()
552 lru = page_lru_base_type(page); in lru_deactivate_file_fn()
554 del_page_from_lru_list(page, lruvec, lru + active); in lru_deactivate_file_fn()
555 ClearPageActive(page); in lru_deactivate_file_fn()
556 ClearPageReferenced(page); in lru_deactivate_file_fn()
558 if (PageWriteback(page) || PageDirty(page)) { in lru_deactivate_file_fn()
564 add_page_to_lru_list(page, lruvec, lru); in lru_deactivate_file_fn()
565 SetPageReclaim(page); in lru_deactivate_file_fn()
571 add_page_to_lru_list_tail(page, lruvec, lru); in lru_deactivate_file_fn()
582 static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, in lru_deactivate_fn() argument
585 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { in lru_deactivate_fn()
586 int lru = page_lru_base_type(page); in lru_deactivate_fn()
587 int nr_pages = thp_nr_pages(page); in lru_deactivate_fn()
589 del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); in lru_deactivate_fn()
590 ClearPageActive(page); in lru_deactivate_fn()
591 ClearPageReferenced(page); in lru_deactivate_fn()
592 add_page_to_lru_list(page, lruvec, lru); in lru_deactivate_fn()
600 static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, in lru_lazyfree_fn() argument
603 if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && in lru_lazyfree_fn()
604 !PageSwapCache(page) && !PageUnevictable(page)) { in lru_lazyfree_fn()
605 bool active = PageActive(page); in lru_lazyfree_fn()
606 int nr_pages = thp_nr_pages(page); in lru_lazyfree_fn()
608 del_page_from_lru_list(page, lruvec, in lru_lazyfree_fn()
610 ClearPageActive(page); in lru_lazyfree_fn()
611 ClearPageReferenced(page); in lru_lazyfree_fn()
617 ClearPageSwapBacked(page); in lru_lazyfree_fn()
618 add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); in lru_lazyfree_fn()
672 void deactivate_file_page(struct page *page) in deactivate_file_page() argument
678 if (PageUnevictable(page)) in deactivate_file_page()
681 if (likely(get_page_unless_zero(page))) { in deactivate_file_page()
687 if (!pagevec_add(pvec, page) || PageCompound(page)) in deactivate_file_page()
701 void deactivate_page(struct page *page) in deactivate_page() argument
703 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { in deactivate_page()
708 get_page(page); in deactivate_page()
709 if (!pagevec_add(pvec, page) || PageCompound(page)) in deactivate_page()
722 void mark_page_lazyfree(struct page *page) in mark_page_lazyfree() argument
724 if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && in mark_page_lazyfree()
725 !PageSwapCache(page) && !PageUnevictable(page)) { in mark_page_lazyfree()
730 get_page(page); in mark_page_lazyfree()
731 if (!pagevec_add(pvec, page) || PageCompound(page)) in mark_page_lazyfree()
876 void release_pages(struct page **pages, int nr) in release_pages()
886 struct page *page = pages[i]; in release_pages() local
898 page = compound_head(page); in release_pages()
899 if (is_huge_zero_page(page)) in release_pages()
902 if (is_zone_device_page(page)) { in release_pages()
914 if (page_is_devmap_managed(page)) { in release_pages()
915 put_devmap_managed_page(page); in release_pages()
920 if (!put_page_testzero(page)) in release_pages()
923 if (PageCompound(page)) { in release_pages()
928 __put_compound_page(page); in release_pages()
932 if (PageLRU(page)) { in release_pages()
933 struct pglist_data *pgdat = page_pgdat(page); in release_pages()
944 lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); in release_pages()
945 VM_BUG_ON_PAGE(!PageLRU(page), page); in release_pages()
946 __ClearPageLRU(page); in release_pages()
947 del_page_from_lru_list(page, lruvec, page_off_lru(page)); in release_pages()
950 __ClearPageWaiters(page); in release_pages()
952 list_add(&page->lru, &pages_to_free); in release_pages()
985 void lru_add_page_tail(struct page *page, struct page *page_tail, in lru_add_page_tail() argument
988 VM_BUG_ON_PAGE(!PageHead(page), page); in lru_add_page_tail()
989 VM_BUG_ON_PAGE(PageCompound(page_tail), page); in lru_add_page_tail()
990 VM_BUG_ON_PAGE(PageLRU(page_tail), page); in lru_add_page_tail()
996 if (likely(PageLRU(page))) in lru_add_page_tail()
997 list_add_tail(&page_tail->lru, &page->lru); in lru_add_page_tail()
1016 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, in __pagevec_lru_add_fn() argument
1020 int was_unevictable = TestClearPageUnevictable(page); in __pagevec_lru_add_fn()
1021 int nr_pages = thp_nr_pages(page); in __pagevec_lru_add_fn()
1023 VM_BUG_ON_PAGE(PageLRU(page), page); in __pagevec_lru_add_fn()
1051 SetPageLRU(page); in __pagevec_lru_add_fn()
1054 if (page_evictable(page)) { in __pagevec_lru_add_fn()
1055 lru = page_lru(page); in __pagevec_lru_add_fn()
1060 ClearPageActive(page); in __pagevec_lru_add_fn()
1061 SetPageUnevictable(page); in __pagevec_lru_add_fn()
1066 add_page_to_lru_list(page, lruvec, lru); in __pagevec_lru_add_fn()
1067 trace_mm_lru_insertion(page, lru); in __pagevec_lru_add_fn()
1127 struct page *page = pvec->pages[i]; in pagevec_remove_exceptionals() local
1128 if (!xa_is_value(page)) in pagevec_remove_exceptionals()
1129 pvec->pages[j++] = page; in pagevec_remove_exceptionals()
1201 void put_devmap_managed_page(struct page *page) in put_devmap_managed_page() argument
1205 if (WARN_ON_ONCE(!page_is_devmap_managed(page))) in put_devmap_managed_page()
1208 count = page_ref_dec_return(page); in put_devmap_managed_page()
1216 free_devmap_managed_page(page); in put_devmap_managed_page()
1218 __put_page(page); in put_devmap_managed_page()