10 static inline bool not_found(struct page_vma_mapped_walk *pvmw)  in not_found()  argument
12 	page_vma_mapped_walk_done(pvmw);  in not_found()
16 static bool map_pte(struct page_vma_mapped_walk *pvmw)  in map_pte()  argument
18 	pvmw->pte = pte_offset_map(pvmw->pmd, pvmw->address);  in map_pte()
19 	if (!(pvmw->flags & PVMW_SYNC)) {  in map_pte()
20 		if (pvmw->flags & PVMW_MIGRATION) {  in map_pte()
21 			if (!is_swap_pte(*pvmw->pte))  in map_pte()
39 			if (is_swap_pte(*pvmw->pte)) {  in map_pte()
43 				entry = pte_to_swp_entry(*pvmw->pte);  in map_pte()
46 			} else if (!pte_present(*pvmw->pte))  in map_pte()
50 	pvmw->ptl = pte_lockptr(pvmw->vma->vm_mm, pvmw->pmd);  in map_pte()
51 	spin_lock(pvmw->ptl);  in map_pte()
81 static bool check_pte(struct page_vma_mapped_walk *pvmw)  in check_pte()  argument
85 	if (pvmw->flags & PVMW_MIGRATION) {  in check_pte()
87 		if (!is_swap_pte(*pvmw->pte))  in check_pte()
89 		entry = pte_to_swp_entry(*pvmw->pte);  in check_pte()
95 	} else if (is_swap_pte(*pvmw->pte)) {  in check_pte()
99 		entry = pte_to_swp_entry(*pvmw->pte);  in check_pte()
105 		if (!pte_present(*pvmw->pte))  in check_pte()
108 		pfn = pte_pfn(*pvmw->pte);  in check_pte()
111 	return pfn_in_hpage(pvmw->page, pfn);  in check_pte()
138 bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)  in page_vma_mapped_walk()  argument
140 	struct mm_struct *mm = pvmw->vma->vm_mm;  in page_vma_mapped_walk()
141 	struct page *page = pvmw->page;  in page_vma_mapped_walk()
148 	if (pvmw->pmd && !pvmw->pte)  in page_vma_mapped_walk()
149 		return not_found(pvmw);  in page_vma_mapped_walk()
151 	if (pvmw->pte)  in page_vma_mapped_walk()
154 	if (unlikely(PageHuge(pvmw->page))) {  in page_vma_mapped_walk()
156 		pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page));  in page_vma_mapped_walk()
157 		if (!pvmw->pte)  in page_vma_mapped_walk()
160 		pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte);  in page_vma_mapped_walk()
161 		spin_lock(pvmw->ptl);  in page_vma_mapped_walk()
162 		if (!check_pte(pvmw))  in page_vma_mapped_walk()
163 			return not_found(pvmw);  in page_vma_mapped_walk()
167 	pgd = pgd_offset(mm, pvmw->address);  in page_vma_mapped_walk()
170 	p4d = p4d_offset(pgd, pvmw->address);  in page_vma_mapped_walk()
173 	pud = pud_offset(p4d, pvmw->address);  in page_vma_mapped_walk()
176 	pvmw->pmd = pmd_offset(pud, pvmw->address);  in page_vma_mapped_walk()
182 	pmde = READ_ONCE(*pvmw->pmd);  in page_vma_mapped_walk()
184 		pvmw->ptl = pmd_lock(mm, pvmw->pmd);  in page_vma_mapped_walk()
185 		if (likely(pmd_trans_huge(*pvmw->pmd))) {  in page_vma_mapped_walk()
186 			if (pvmw->flags & PVMW_MIGRATION)  in page_vma_mapped_walk()
187 				return not_found(pvmw);  in page_vma_mapped_walk()
188 			if (pmd_page(*pvmw->pmd) != page)  in page_vma_mapped_walk()
189 				return not_found(pvmw);  in page_vma_mapped_walk()
191 		} else if (!pmd_present(*pvmw->pmd)) {  in page_vma_mapped_walk()
193 				if (!(pvmw->flags & PVMW_MIGRATION))  in page_vma_mapped_walk()
194 					return not_found(pvmw);  in page_vma_mapped_walk()
195 				if (is_migration_entry(pmd_to_swp_entry(*pvmw->pmd))) {  in page_vma_mapped_walk()
196 					swp_entry_t entry = pmd_to_swp_entry(*pvmw->pmd);  in page_vma_mapped_walk()
199 						return not_found(pvmw);  in page_vma_mapped_walk()
203 			return not_found(pvmw);  in page_vma_mapped_walk()
206 			spin_unlock(pvmw->ptl);  in page_vma_mapped_walk()
207 			pvmw->ptl = NULL;  in page_vma_mapped_walk()
212 	if (!map_pte(pvmw))  in page_vma_mapped_walk()
215 		if (check_pte(pvmw))  in page_vma_mapped_walk()
219 		if (!PageTransHuge(pvmw->page) || PageHuge(pvmw->page))  in page_vma_mapped_walk()
220 			return not_found(pvmw);  in page_vma_mapped_walk()
222 			pvmw->address += PAGE_SIZE;  in page_vma_mapped_walk()
223 			if (pvmw->address >= pvmw->vma->vm_end ||  in page_vma_mapped_walk()
224 			    pvmw->address >=  in page_vma_mapped_walk()
225 					__vma_address(pvmw->page, pvmw->vma) +  in page_vma_mapped_walk()
226 					hpage_nr_pages(pvmw->page) * PAGE_SIZE)  in page_vma_mapped_walk()
227 				return not_found(pvmw);  in page_vma_mapped_walk()
229 			if (pvmw->address % PMD_SIZE == 0) {  in page_vma_mapped_walk()
230 				pte_unmap(pvmw->pte);  in page_vma_mapped_walk()
231 				if (pvmw->ptl) {  in page_vma_mapped_walk()
232 					spin_unlock(pvmw->ptl);  in page_vma_mapped_walk()
233 					pvmw->ptl = NULL;  in page_vma_mapped_walk()
237 				pvmw->pte++;  in page_vma_mapped_walk()
239 		} while (pte_none(*pvmw->pte));  in page_vma_mapped_walk()
241 		if (!pvmw->ptl) {  in page_vma_mapped_walk()
242 			pvmw->ptl = pte_lockptr(mm, pvmw->pmd);  in page_vma_mapped_walk()
243 			spin_lock(pvmw->ptl);  in page_vma_mapped_walk()
259 	struct page_vma_mapped_walk pvmw = {  in page_mapped_in_vma()  local
271 	pvmw.address = max(start, vma->vm_start);  in page_mapped_in_vma()
272 	if (!page_vma_mapped_walk(&pvmw))  in page_mapped_in_vma()
274 	page_vma_mapped_walk_done(&pvmw);  in page_mapped_in_vma()