Lines Matching full:page
10 * Provides methods for unmapping each kind of mapped page:
25 * page->flags PG_locked (lock_page) * (see huegtlbfs below)
28 * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
50 * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
51 * page->flags PG_locked (lock_page)
274 * searches where page is mapped.
466 * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
470 * have been relevant to this page.
472 * The page might have been remapped to a different anon_vma or the anon_vma
477 * ensure that any anon_vma obtained from the page will still be valid for as
481 * chain and verify that the page in question is indeed mapped in it
485 * page_remove_rmap() that the anon_vma pointer from page->mapping is valid
489 struct anon_vma *page_get_anon_vma(struct page *page) in page_get_anon_vma() argument
495 anon_mapping = (unsigned long)READ_ONCE(page->mapping); in page_get_anon_vma()
498 if (!page_mapped(page)) in page_get_anon_vma()
508 * If this page is still mapped, then its anon_vma cannot have been in page_get_anon_vma()
514 if (!page_mapped(page)) { in page_get_anon_vma()
532 struct anon_vma *page_lock_anon_vma_read(struct page *page) in page_lock_anon_vma_read() argument
539 anon_mapping = (unsigned long)READ_ONCE(page->mapping); in page_lock_anon_vma_read()
542 if (!page_mapped(page)) in page_lock_anon_vma_read()
549 * If the page is still mapped, then this anon_vma is still in page_lock_anon_vma_read()
553 if (!page_mapped(page)) { in page_lock_anon_vma_read()
566 if (!page_mapped(page)) { in page_lock_anon_vma_read()
603 * before any IO is initiated on the page to prevent lost writes. Similarly,
644 * before the page is queued for IO. in set_tlb_ubc_flush_pending()
673 * the page and flushing the page. If this race occurs, it potentially allows
709 * At what user virtual address is page expected in vma?
710 * Caller should check the page is actually part of the vma.
712 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) in page_address_in_vma() argument
714 if (PageAnon(page)) { in page_address_in_vma()
715 struct anon_vma *page__anon_vma = page_anon_vma(page); in page_address_in_vma()
725 } else if (vma->vm_file->f_mapping != compound_head(page)->mapping) { in page_address_in_vma()
729 return vma_address(page, vma); in page_address_in_vma()
775 static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, in page_referenced_one() argument
780 .page = page, in page_referenced_one()
805 * If the page has been used in another mapping, in page_referenced_one()
808 * PG_referenced or activated the page. in page_referenced_one()
818 /* unexpected pmd-mapped page? */ in page_referenced_one()
826 clear_page_idle(page); in page_referenced_one()
827 if (test_and_clear_page_young(page)) in page_referenced_one()
853 * page_referenced - test if the page was referenced
854 * @page: the page to test
855 * @is_locked: caller holds lock on the page
857 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
859 * Quick test_and_clear_referenced for all mappings to a page,
860 * returns the number of ptes which referenced the page.
862 int page_referenced(struct page *page, in page_referenced() argument
869 .mapcount = total_mapcount(page), in page_referenced()
883 if (!page_rmapping(page)) in page_referenced()
886 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { in page_referenced()
887 we_locked = trylock_page(page); in page_referenced()
901 rmap_walk(page, &rwc); in page_referenced()
905 unlock_page(page); in page_referenced()
910 static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, in page_mkclean_one() argument
914 .page = page, in page_mkclean_one()
924 * the page can not be free from this function. in page_mkclean_one()
928 vma_address_end(page, vma)); in page_mkclean_one()
956 flush_cache_page(vma, address, page_to_pfn(page)); in page_mkclean_one()
963 /* unexpected pmd-mapped page? */ in page_mkclean_one()
970 * downgrading page table protection not changing it to point in page_mkclean_one()
971 * to a new page. in page_mkclean_one()
992 int page_mkclean(struct page *page) in page_mkclean() argument
1002 BUG_ON(!PageLocked(page)); in page_mkclean()
1004 if (!page_mapped(page)) in page_mkclean()
1007 mapping = page_mapping(page); in page_mkclean()
1011 rmap_walk(page, &rwc); in page_mkclean()
1018 * page_move_anon_rmap - move a page to our anon_vma
1019 * @page: the page to move to our anon_vma
1020 * @vma: the vma the page belongs to
1022 * When a page belongs exclusively to one process after a COW event,
1023 * that page can be moved into the anon_vma that belongs to just that
1027 void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) in page_move_anon_rmap() argument
1031 page = compound_head(page); in page_move_anon_rmap()
1033 VM_BUG_ON_PAGE(!PageLocked(page), page); in page_move_anon_rmap()
1042 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); in page_move_anon_rmap()
1047 * @page: Page or Hugepage to add to rmap
1048 * @vma: VM area to add page to.
1050 * @exclusive: the page is exclusively owned by the current process
1052 static void __page_set_anon_rmap(struct page *page, in __page_set_anon_rmap() argument
1059 if (PageAnon(page)) in __page_set_anon_rmap()
1063 * If the page isn't exclusively mapped into this vma, in __page_set_anon_rmap()
1065 * page mapping! in __page_set_anon_rmap()
1071 page->mapping = (struct address_space *) anon_vma; in __page_set_anon_rmap()
1072 page->index = linear_page_index(vma, address); in __page_set_anon_rmap()
1077 * @page: the page to add the mapping to
1081 static void __page_check_anon_rmap(struct page *page, in __page_check_anon_rmap() argument
1085 * The page's anon-rmap details (mapping and index) are guaranteed to in __page_check_anon_rmap()
1089 * always holds the page locked, except if called from page_dup_rmap, in __page_check_anon_rmap()
1090 * in which case the page is already known to be setup. in __page_check_anon_rmap()
1096 VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page); in __page_check_anon_rmap()
1097 VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address), in __page_check_anon_rmap()
1098 page); in __page_check_anon_rmap()
1102 * page_add_anon_rmap - add pte mapping to an anonymous page
1103 * @page: the page to add the mapping to
1106 * @compound: charge the page as compound or small page
1108 * The caller needs to hold the pte lock, and the page must be locked in
1113 void page_add_anon_rmap(struct page *page, in page_add_anon_rmap() argument
1116 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); in page_add_anon_rmap()
1124 void do_page_add_anon_rmap(struct page *page, in do_page_add_anon_rmap() argument
1130 if (unlikely(PageKsm(page))) in do_page_add_anon_rmap()
1131 lock_page_memcg(page); in do_page_add_anon_rmap()
1133 VM_BUG_ON_PAGE(!PageLocked(page), page); in do_page_add_anon_rmap()
1137 VM_BUG_ON_PAGE(!PageLocked(page), page); in do_page_add_anon_rmap()
1138 VM_BUG_ON_PAGE(!PageTransHuge(page), page); in do_page_add_anon_rmap()
1139 mapcount = compound_mapcount_ptr(page); in do_page_add_anon_rmap()
1142 first = atomic_inc_and_test(&page->_mapcount); in do_page_add_anon_rmap()
1146 int nr = compound ? thp_nr_pages(page) : 1; in do_page_add_anon_rmap()
1154 __inc_lruvec_page_state(page, NR_ANON_THPS); in do_page_add_anon_rmap()
1155 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); in do_page_add_anon_rmap()
1158 if (unlikely(PageKsm(page))) { in do_page_add_anon_rmap()
1159 unlock_page_memcg(page); in do_page_add_anon_rmap()
1165 __page_set_anon_rmap(page, vma, address, in do_page_add_anon_rmap()
1168 __page_check_anon_rmap(page, vma, address); in do_page_add_anon_rmap()
1172 * page_add_new_anon_rmap - add pte mapping to a new anonymous page
1173 * @page: the page to add the mapping to
1176 * @compound: charge the page as compound or small page
1180 * Page does not have to be locked.
1182 void page_add_new_anon_rmap(struct page *page, in page_add_new_anon_rmap() argument
1185 int nr = compound ? thp_nr_pages(page) : 1; in page_add_new_anon_rmap()
1188 __SetPageSwapBacked(page); in page_add_new_anon_rmap()
1190 VM_BUG_ON_PAGE(!PageTransHuge(page), page); in page_add_new_anon_rmap()
1192 atomic_set(compound_mapcount_ptr(page), 0); in page_add_new_anon_rmap()
1193 if (hpage_pincount_available(page)) in page_add_new_anon_rmap()
1194 atomic_set(compound_pincount_ptr(page), 0); in page_add_new_anon_rmap()
1196 __inc_lruvec_page_state(page, NR_ANON_THPS); in page_add_new_anon_rmap()
1199 VM_BUG_ON_PAGE(PageTransCompound(page), page); in page_add_new_anon_rmap()
1201 atomic_set(&page->_mapcount, 0); in page_add_new_anon_rmap()
1203 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); in page_add_new_anon_rmap()
1204 __page_set_anon_rmap(page, vma, address, 1); in page_add_new_anon_rmap()
1208 * page_add_file_rmap - add pte mapping to a file page
1209 * @page: the page to add the mapping to
1210 * @compound: charge the page as compound or small page
1214 void page_add_file_rmap(struct page *page, bool compound) in page_add_file_rmap() argument
1218 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); in page_add_file_rmap()
1219 lock_page_memcg(page); in page_add_file_rmap()
1220 if (compound && PageTransHuge(page)) { in page_add_file_rmap()
1221 for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { in page_add_file_rmap()
1222 if (atomic_inc_and_test(&page[i]._mapcount)) in page_add_file_rmap()
1225 if (!atomic_inc_and_test(compound_mapcount_ptr(page))) in page_add_file_rmap()
1227 if (PageSwapBacked(page)) in page_add_file_rmap()
1228 __inc_node_page_state(page, NR_SHMEM_PMDMAPPED); in page_add_file_rmap()
1230 __inc_node_page_state(page, NR_FILE_PMDMAPPED); in page_add_file_rmap()
1232 if (PageTransCompound(page) && page_mapping(page)) { in page_add_file_rmap()
1233 VM_WARN_ON_ONCE(!PageLocked(page)); in page_add_file_rmap()
1235 SetPageDoubleMap(compound_head(page)); in page_add_file_rmap()
1236 if (PageMlocked(page)) in page_add_file_rmap()
1237 clear_page_mlock(compound_head(page)); in page_add_file_rmap()
1239 if (!atomic_inc_and_test(&page->_mapcount)) in page_add_file_rmap()
1242 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr); in page_add_file_rmap()
1244 unlock_page_memcg(page); in page_add_file_rmap()
1247 static void page_remove_file_rmap(struct page *page, bool compound) in page_remove_file_rmap() argument
1251 VM_BUG_ON_PAGE(compound && !PageHead(page), page); in page_remove_file_rmap()
1254 if (unlikely(PageHuge(page))) { in page_remove_file_rmap()
1256 atomic_dec(compound_mapcount_ptr(page)); in page_remove_file_rmap()
1260 /* page still mapped by someone else? */ in page_remove_file_rmap()
1261 if (compound && PageTransHuge(page)) { in page_remove_file_rmap()
1262 for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { in page_remove_file_rmap()
1263 if (atomic_add_negative(-1, &page[i]._mapcount)) in page_remove_file_rmap()
1266 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) in page_remove_file_rmap()
1268 if (PageSwapBacked(page)) in page_remove_file_rmap()
1269 __dec_node_page_state(page, NR_SHMEM_PMDMAPPED); in page_remove_file_rmap()
1271 __dec_node_page_state(page, NR_FILE_PMDMAPPED); in page_remove_file_rmap()
1273 if (!atomic_add_negative(-1, &page->_mapcount)) in page_remove_file_rmap()
1282 __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr); in page_remove_file_rmap()
1284 if (unlikely(PageMlocked(page))) in page_remove_file_rmap()
1285 clear_page_mlock(page); in page_remove_file_rmap()
1288 static void page_remove_anon_compound_rmap(struct page *page) in page_remove_anon_compound_rmap() argument
1292 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) in page_remove_anon_compound_rmap()
1296 if (unlikely(PageHuge(page))) in page_remove_anon_compound_rmap()
1302 __dec_lruvec_page_state(page, NR_ANON_THPS); in page_remove_anon_compound_rmap()
1304 if (TestClearPageDoubleMap(page)) { in page_remove_anon_compound_rmap()
1309 for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { in page_remove_anon_compound_rmap()
1310 if (atomic_add_negative(-1, &page[i]._mapcount)) in page_remove_anon_compound_rmap()
1315 * Queue the page for deferred split if at least one small in page_remove_anon_compound_rmap()
1316 * page of the compound page is unmapped, but at least one in page_remove_anon_compound_rmap()
1317 * small page is still mapped. in page_remove_anon_compound_rmap()
1319 if (nr && nr < thp_nr_pages(page)) in page_remove_anon_compound_rmap()
1320 deferred_split_huge_page(page); in page_remove_anon_compound_rmap()
1322 nr = thp_nr_pages(page); in page_remove_anon_compound_rmap()
1325 if (unlikely(PageMlocked(page))) in page_remove_anon_compound_rmap()
1326 clear_page_mlock(page); in page_remove_anon_compound_rmap()
1329 __mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr); in page_remove_anon_compound_rmap()
1333 * page_remove_rmap - take down pte mapping from a page
1334 * @page: page to remove mapping from
1335 * @compound: uncharge the page as compound or small page
1339 void page_remove_rmap(struct page *page, bool compound) in page_remove_rmap() argument
1341 lock_page_memcg(page); in page_remove_rmap()
1343 if (!PageAnon(page)) { in page_remove_rmap()
1344 page_remove_file_rmap(page, compound); in page_remove_rmap()
1349 page_remove_anon_compound_rmap(page); in page_remove_rmap()
1353 /* page still mapped by someone else? */ in page_remove_rmap()
1354 if (!atomic_add_negative(-1, &page->_mapcount)) in page_remove_rmap()
1362 __dec_lruvec_page_state(page, NR_ANON_MAPPED); in page_remove_rmap()
1364 if (unlikely(PageMlocked(page))) in page_remove_rmap()
1365 clear_page_mlock(page); in page_remove_rmap()
1367 if (PageTransCompound(page)) in page_remove_rmap()
1368 deferred_split_huge_page(compound_head(page)); in page_remove_rmap()
1380 unlock_page_memcg(page); in page_remove_rmap()
1386 static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, in try_to_unmap_one() argument
1391 .page = page, in try_to_unmap_one()
1396 struct page *subpage; in try_to_unmap_one()
1405 * if page table locking is skipped: use TTU_SYNC to wait for that. in try_to_unmap_one()
1415 is_zone_device_page(page) && !is_device_private_page(page)) in try_to_unmap_one()
1420 flags & TTU_SPLIT_FREEZE, page); in try_to_unmap_one()
1428 * Note that the page can not be free in this function as call of in try_to_unmap_one()
1429 * try_to_unmap() must hold a reference on the page. in try_to_unmap_one()
1431 range.end = PageKsm(page) ? in try_to_unmap_one()
1432 address + PAGE_SIZE : vma_address_end(page, vma); in try_to_unmap_one()
1435 if (PageHuge(page)) { in try_to_unmap_one()
1449 VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); in try_to_unmap_one()
1451 set_pmd_migration_entry(&pvmw, page); in try_to_unmap_one()
1464 * If the page is mlock()d, we cannot swap it out. in try_to_unmap_one()
1471 if (!PageTransCompound(page)) { in try_to_unmap_one()
1476 mlock_vma_page(page); in try_to_unmap_one()
1487 VM_BUG_ON_PAGE(!pvmw.pte, page); in try_to_unmap_one()
1489 subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte); in try_to_unmap_one()
1492 if (PageHuge(page) && !PageAnon(page)) { in try_to_unmap_one()
1502 * page. There is no way of knowing exactly in try_to_unmap_one()
1513 * The ref count of the PMD page was dropped in try_to_unmap_one()
1518 * unmap the actual page and drop map count in try_to_unmap_one()
1528 is_zone_device_page(page)) { in try_to_unmap_one()
1535 * Store the pfn of the page in a special migration in try_to_unmap_one()
1539 entry = make_migration_entry(page, 0); in try_to_unmap_one()
1543 * pteval maps a zone device page and is therefore in try_to_unmap_one()
1558 * migrated, just set it to page. This will need to be in try_to_unmap_one()
1562 subpage = page; in try_to_unmap_one()
1566 /* Nuke the page table entry. */ in try_to_unmap_one()
1571 * a remote CPU could still be writing to the page. in try_to_unmap_one()
1584 /* Move the dirty bit to the page. Now the pte is gone. */ in try_to_unmap_one()
1586 set_page_dirty(page); in try_to_unmap_one()
1591 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { in try_to_unmap_one()
1593 if (PageHuge(page)) { in try_to_unmap_one()
1594 hugetlb_count_sub(compound_nr(page), mm); in try_to_unmap_one()
1599 dec_mm_counter(mm, mm_counter(page)); in try_to_unmap_one()
1608 * The guest indicated that the page content is of no in try_to_unmap_one()
1612 * page. When userfaultfd is active, we must not drop in try_to_unmap_one()
1613 * this page though, as its main user (postcopy in try_to_unmap_one()
1617 dec_mm_counter(mm, mm_counter(page)); in try_to_unmap_one()
1634 * Store the pfn of the page in a special migration in try_to_unmap_one()
1650 } else if (PageAnon(page)) { in try_to_unmap_one()
1657 if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) { in try_to_unmap_one()
1667 /* MADV_FREE page check */ in try_to_unmap_one()
1668 if (!PageSwapBacked(page)) { in try_to_unmap_one()
1678 ref_count = page_ref_count(page); in try_to_unmap_one()
1679 map_count = page_mapcount(page); in try_to_unmap_one()
1682 * Order reads for page refcount and dirty flag in try_to_unmap_one()
1688 * The only page refs must be one from isolation in try_to_unmap_one()
1692 !PageDirty(page)) { in try_to_unmap_one()
1701 * If the page was redirtied, it cannot be in try_to_unmap_one()
1702 * discarded. Remap the page to page table. in try_to_unmap_one()
1705 SetPageSwapBacked(page); in try_to_unmap_one()
1742 * This is a locked file-backed page, thus it cannot in try_to_unmap_one()
1743 * be removed from the page cache and replaced by a new in try_to_unmap_one()
1744 * page before mmu_notifier_invalidate_range_end, so no in try_to_unmap_one()
1745 * concurrent thread might update its page table to in try_to_unmap_one()
1746 * point at new page while a device still is using this in try_to_unmap_one()
1747 * page. in try_to_unmap_one()
1751 dec_mm_counter(mm, mm_counter_file(page)); in try_to_unmap_one()
1756 * done above for all cases requiring it to happen under page in try_to_unmap_one()
1761 page_remove_rmap(subpage, PageHuge(page)); in try_to_unmap_one()
1762 put_page(page); in try_to_unmap_one()
1775 static int page_not_mapped(struct page *page) in page_not_mapped() argument
1777 return !page_mapped(page); in page_not_mapped()
1781 * try_to_unmap - try to remove all page table mappings to a page
1782 * @page: the page to get unmapped
1785 * Tries to remove all the page table entries which are mapping this
1786 * page, used in the pageout path. Caller must hold the page lock.
1790 bool try_to_unmap(struct page *page, enum ttu_flags flags) in try_to_unmap() argument
1802 * page tables leading to a race where migration cannot in try_to_unmap()
1808 && !PageKsm(page) && PageAnon(page)) in try_to_unmap()
1812 rmap_walk_locked(page, &rwc); in try_to_unmap()
1814 rmap_walk(page, &rwc); in try_to_unmap()
1820 * if page table locking is skipped: use TTU_SYNC to wait for that. in try_to_unmap()
1822 return !page_mapcount(page); in try_to_unmap()
1826 * try_to_munlock - try to munlock a page
1827 * @page: the page to be munlocked
1829 * Called from munlock code. Checks all of the VMAs mapping the page
1830 * to make sure nobody else has this page mlocked. The page will be
1834 void try_to_munlock(struct page *page) in try_to_munlock() argument
1844 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); in try_to_munlock()
1845 VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page); in try_to_munlock()
1847 rmap_walk(page, &rwc); in try_to_munlock()
1859 static struct anon_vma *rmap_walk_anon_lock(struct page *page, in rmap_walk_anon_lock() argument
1865 return rwc->anon_lock(page); in rmap_walk_anon_lock()
1873 anon_vma = page_anon_vma(page); in rmap_walk_anon_lock()
1882 * rmap_walk_anon - do something to anonymous page using the object-based
1884 * @page: the page to be handled
1887 * Find all the mappings of a page using the mapping pointer and the vma chains
1891 * where the page was found will be held for write. So, we won't recheck
1895 static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, in rmap_walk_anon() argument
1903 anon_vma = page_anon_vma(page); in rmap_walk_anon()
1905 VM_BUG_ON_PAGE(!anon_vma, page); in rmap_walk_anon()
1907 anon_vma = rmap_walk_anon_lock(page, rwc); in rmap_walk_anon()
1912 pgoff_start = page_to_pgoff(page); in rmap_walk_anon()
1913 pgoff_end = pgoff_start + thp_nr_pages(page) - 1; in rmap_walk_anon()
1917 unsigned long address = vma_address(page, vma); in rmap_walk_anon()
1925 if (!rwc->rmap_one(page, vma, address, rwc->arg)) in rmap_walk_anon()
1927 if (rwc->done && rwc->done(page)) in rmap_walk_anon()
1936 * rmap_walk_file - do something to file page using the object-based rmap method
1937 * @page: the page to be handled
1940 * Find all the mappings of a page using the mapping pointer and the vma chains
1944 * where the page was found will be held for write. So, we won't recheck
1948 static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, in rmap_walk_file() argument
1951 struct address_space *mapping = page_mapping(page); in rmap_walk_file()
1956 * The page lock not only makes sure that page->mapping cannot in rmap_walk_file()
1961 VM_BUG_ON_PAGE(!PageLocked(page), page); in rmap_walk_file()
1966 pgoff_start = page_to_pgoff(page); in rmap_walk_file()
1967 pgoff_end = pgoff_start + thp_nr_pages(page) - 1; in rmap_walk_file()
1972 unsigned long address = vma_address(page, vma); in rmap_walk_file()
1980 if (!rwc->rmap_one(page, vma, address, rwc->arg)) in rmap_walk_file()
1982 if (rwc->done && rwc->done(page)) in rmap_walk_file()
1991 void rmap_walk(struct page *page, struct rmap_walk_control *rwc) in rmap_walk() argument
1993 if (unlikely(PageKsm(page))) in rmap_walk()
1994 rmap_walk_ksm(page, rwc); in rmap_walk()
1995 else if (PageAnon(page)) in rmap_walk()
1996 rmap_walk_anon(page, rwc, false); in rmap_walk()
1998 rmap_walk_file(page, rwc, false); in rmap_walk()
2002 void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc) in rmap_walk_locked() argument
2005 VM_BUG_ON_PAGE(PageKsm(page), page); in rmap_walk_locked()
2006 if (PageAnon(page)) in rmap_walk_locked()
2007 rmap_walk_anon(page, rwc, true); in rmap_walk_locked()
2009 rmap_walk_file(page, rwc, true); in rmap_walk_locked()
2018 void hugepage_add_anon_rmap(struct page *page, in hugepage_add_anon_rmap() argument
2024 BUG_ON(!PageLocked(page)); in hugepage_add_anon_rmap()
2027 first = atomic_inc_and_test(compound_mapcount_ptr(page)); in hugepage_add_anon_rmap()
2029 __page_set_anon_rmap(page, vma, address, 0); in hugepage_add_anon_rmap()
2032 void hugepage_add_new_anon_rmap(struct page *page, in hugepage_add_new_anon_rmap() argument
2036 atomic_set(compound_mapcount_ptr(page), 0); in hugepage_add_new_anon_rmap()
2037 if (hpage_pincount_available(page)) in hugepage_add_new_anon_rmap()
2038 atomic_set(compound_pincount_ptr(page), 0); in hugepage_add_new_anon_rmap()
2040 __page_set_anon_rmap(page, vma, address, 1); in hugepage_add_new_anon_rmap()