564 static int smaps_pte_hole(unsigned long addr, unsigned long end,  in smaps_pte_hole()  argument
570 			walk->vma->vm_file->f_mapping, addr, end);  in smaps_pte_hole()
578 static void smaps_pte_entry(pte_t *pte, unsigned long addr,  in smaps_pte_entry()  argument
587 		page = vm_normal_page(vma, addr, *pte);  in smaps_pte_entry()
611 						linear_page_index(vma, addr));  in smaps_pte_entry()
630 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,  in smaps_pmd_entry()  argument
639 	page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);  in smaps_pmd_entry()
653 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,  in smaps_pmd_entry()  argument
659 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,  in smaps_pte_range()  argument
669 			smaps_pmd_entry(pmd, addr, walk);  in smaps_pte_range()
681 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in smaps_pte_range()
682 	for (; addr != end; pte++, addr += PAGE_SIZE)  in smaps_pte_range()
683 		smaps_pte_entry(pte, addr, walk);  in smaps_pte_range()
765 				 unsigned long addr, unsigned long end,  in smaps_hugetlb_range()  argument
773 		page = vm_normal_page(vma, addr, *pte);  in smaps_hugetlb_range()
1051 		unsigned long addr, pte_t *pte)  in clear_soft_dirty()  argument
1064 		old_pte = ptep_modify_prot_start(vma, addr, pte);  in clear_soft_dirty()
1067 		ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);  in clear_soft_dirty()
1070 		set_pte_at(vma->vm_mm, addr, pte, ptent);  in clear_soft_dirty()
1075 		unsigned long addr, pte_t *pte)  in clear_soft_dirty()  argument
1082 		unsigned long addr, pmd_t *pmdp)  in clear_soft_dirty_pmd()  argument
1088 		old = pmdp_invalidate(vma, addr, pmdp);  in clear_soft_dirty_pmd()
1097 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);  in clear_soft_dirty_pmd()
1100 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);  in clear_soft_dirty_pmd()
1105 		unsigned long addr, pmd_t *pmdp)  in clear_soft_dirty_pmd()  argument
1110 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,  in clear_refs_pte_range()  argument
1122 			clear_soft_dirty_pmd(vma, addr, pmd);  in clear_refs_pte_range()
1132 		pmdp_test_and_clear_young(vma, addr, pmd);  in clear_refs_pte_range()
1143 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in clear_refs_pte_range()
1144 	for (; addr != end; pte++, addr += PAGE_SIZE) {  in clear_refs_pte_range()
1148 			clear_soft_dirty(vma, addr, pte);  in clear_refs_pte_range()
1155 		page = vm_normal_page(vma, addr, ptent);  in clear_refs_pte_range()
1160 		ptep_test_and_clear_young(vma, addr, pte);  in clear_refs_pte_range()
1337 static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,  in add_to_pagemap()  argument
1350 	unsigned long addr = start;  in pagemap_pte_hole()  local
1353 	while (addr < end) {  in pagemap_pte_hole()
1354 		struct vm_area_struct *vma = find_vma(walk->mm, addr);  in pagemap_pte_hole()
1364 		for (; addr < hole_end; addr += PAGE_SIZE) {  in pagemap_pte_hole()
1365 			err = add_to_pagemap(addr, &pme, pm);  in pagemap_pte_hole()
1376 		for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {  in pagemap_pte_hole()
1377 			err = add_to_pagemap(addr, &pme, pm);  in pagemap_pte_hole()
1387 		struct vm_area_struct *vma, unsigned long addr, pte_t pte)  in pte_to_pagemap_entry()  argument
1396 		page = vm_normal_page(vma, addr, pte);  in pte_to_pagemap_entry()
1425 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,  in pagemap_pmd_range()  argument
1452 					((addr & ~PMD_MASK) >> PAGE_SHIFT);  in pagemap_pmd_range()
1461 					((addr & ~PMD_MASK) >> PAGE_SHIFT);  in pagemap_pmd_range()
1476 		for (; addr != end; addr += PAGE_SIZE) {  in pagemap_pmd_range()
1479 			err = add_to_pagemap(addr, &pme, pm);  in pagemap_pmd_range()
1501 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);  in pagemap_pmd_range()
1502 	for (; addr < end; pte++, addr += PAGE_SIZE) {  in pagemap_pmd_range()
1505 		pme = pte_to_pagemap_entry(pm, vma, addr, *pte);  in pagemap_pmd_range()
1506 		err = add_to_pagemap(addr, &pme, pm);  in pagemap_pmd_range()
1520 				 unsigned long addr, unsigned long end,  in pagemap_hugetlb_range()  argument
1545 				((addr & ~hmask) >> PAGE_SHIFT);  in pagemap_hugetlb_range()
1548 	for (; addr != end; addr += PAGE_SIZE) {  in pagemap_hugetlb_range()
1551 		err = add_to_pagemap(addr, &pme, pm);  in pagemap_hugetlb_range()
1757 		unsigned long addr)  in can_gather_numa_stats()  argument
1765 	page = vm_normal_page(vma, addr, pte);  in can_gather_numa_stats()
1782 					      unsigned long addr)  in can_gather_numa_stats_pmd()  argument
1790 	page = vm_normal_page_pmd(vma, addr, pmd);  in can_gather_numa_stats_pmd()
1805 static int gather_pte_stats(pmd_t *pmd, unsigned long addr,  in gather_pte_stats()  argument
1819 		page = can_gather_numa_stats_pmd(*pmd, vma, addr);  in gather_pte_stats()
1830 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);  in gather_pte_stats()
1832 		struct page *page = can_gather_numa_stats(*pte, vma, addr);  in gather_pte_stats()
1837 	} while (pte++, addr += PAGE_SIZE, addr != end);  in gather_pte_stats()
1844 		unsigned long addr, unsigned long end, struct mm_walk *walk)  in gather_hugetlb_stats()  argument
1864 		unsigned long addr, unsigned long end, struct mm_walk *walk)  in gather_hugetlb_stats()  argument