214 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,  in free_pte_range()  argument
219 	pte_free_tlb(tlb, token, addr);  in free_pte_range()
220 	mm_dec_nr_ptes(tlb->mm);  in free_pte_range()
223 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,  in free_pmd_range()  argument
237 		free_pte_range(tlb, pmd, addr);  in free_pmd_range()
253 	pmd_free_tlb(tlb, pmd, start);  in free_pmd_range()
254 	mm_dec_nr_pmds(tlb->mm);  in free_pmd_range()
257 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,  in free_pud_range()  argument
271 		free_pmd_range(tlb, pud, addr, next, floor, ceiling);  in free_pud_range()
287 	pud_free_tlb(tlb, pud, start);  in free_pud_range()
288 	mm_dec_nr_puds(tlb->mm);  in free_pud_range()
291 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,  in free_p4d_range()  argument
305 		free_pud_range(tlb, p4d, addr, next, floor, ceiling);  in free_p4d_range()
321 	p4d_free_tlb(tlb, p4d, start);  in free_p4d_range()
327 void free_pgd_range(struct mmu_gather *tlb,  in free_pgd_range()  argument
379 	tlb_change_page_size(tlb, PAGE_SIZE);  in free_pgd_range()
380 	pgd = pgd_offset(tlb->mm, addr);  in free_pgd_range()
385 		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);  in free_pgd_range()
389 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,  in free_pgtables()  argument
404 			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,  in free_pgtables()
417 			free_pgd_range(tlb, addr, vma->vm_end,  in free_pgtables()
1023 static unsigned long zap_pte_range(struct mmu_gather *tlb,  in zap_pte_range()  argument
1028 	struct mm_struct *mm = tlb->mm;  in zap_pte_range()
1036 	tlb_change_page_size(tlb, PAGE_SIZE);  in zap_pte_range()
1066 							tlb->fullmm);  in zap_pte_range()
1067 			tlb_remove_tlb_entry(tlb, pte, addr);  in zap_pte_range()
1084 			if (unlikely(__tlb_remove_page(tlb, page))) {  in zap_pte_range()
1107 			pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);  in zap_pte_range()
1128 		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);  in zap_pte_range()
1136 		tlb_flush_mmu_tlbonly(tlb);  in zap_pte_range()
1147 		tlb_flush_mmu(tlb);  in zap_pte_range()
1158 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,  in zap_pmd_range()  argument
1172 			else if (zap_huge_pmd(tlb, vma, pmd, addr))  in zap_pmd_range()
1185 		next = zap_pte_range(tlb, vma, pmd, addr, next, details);  in zap_pmd_range()
1193 static inline unsigned long zap_pud_range(struct mmu_gather *tlb,  in zap_pud_range()  argument
1206 				VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma);  in zap_pud_range()
1208 			} else if (zap_huge_pud(tlb, vma, pud, addr))  in zap_pud_range()
1214 		next = zap_pmd_range(tlb, vma, pud, addr, next, details);  in zap_pud_range()
1222 static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,  in zap_p4d_range()  argument
1235 		next = zap_pud_range(tlb, vma, p4d, addr, next, details);  in zap_p4d_range()
1241 void unmap_page_range(struct mmu_gather *tlb,  in unmap_page_range()  argument
1250 	tlb_start_vma(tlb, vma);  in unmap_page_range()
1256 		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);  in unmap_page_range()
1258 	tlb_end_vma(tlb, vma);  in unmap_page_range()
1262 static void unmap_single_vma(struct mmu_gather *tlb,  in unmap_single_vma()  argument
1297 				__unmap_hugepage_range_final(tlb, vma, start, end, NULL);  in unmap_single_vma()
1301 			unmap_page_range(tlb, vma, start, end, details);  in unmap_single_vma()
1323 void unmap_vmas(struct mmu_gather *tlb,  in unmap_vmas()  argument
1333 		unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);  in unmap_vmas()
1349 	struct mmu_gather tlb;  in zap_page_range()  local
1354 	tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);  in zap_page_range()
1358 		unmap_single_vma(&tlb, vma, start, range.end, NULL);  in zap_page_range()
1360 	tlb_finish_mmu(&tlb, start, range.end);  in zap_page_range()
1376 	struct mmu_gather tlb;  in zap_page_range_single()  local
1381 	tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);  in zap_page_range_single()
1384 	unmap_single_vma(&tlb, vma, address, range.end, details);  in zap_page_range_single()
1386 	tlb_finish_mmu(&tlb, address, range.end);  in zap_page_range_single()