68 	unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE;  in transparent_hugepage_enabled()  local
70 	if (!transhuge_vma_suitable(vma, addr))  in transparent_hugepage_enabled()
531 		unsigned long addr, unsigned long len,  in __thp_get_unmapped_area()  argument
545 	ret = current->mm->get_unmapped_area(filp, addr, len_pad,  in __thp_get_unmapped_area()
559 	if (ret == addr)  in __thp_get_unmapped_area()
560 		return addr;  in __thp_get_unmapped_area()
566 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,  in thp_get_unmapped_area()  argument
575 	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);  in thp_get_unmapped_area()
579 	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);  in thp_get_unmapped_area()
777 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,  in insert_pfn_pmd()  argument
794 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
795 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
815 	set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
816 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
826 	unsigned long addr = vmf->address & PMD_MASK;  in vmf_insert_pfn_pmd()  local
842 	if (addr < vma->vm_start || addr >= vma->vm_end)  in vmf_insert_pfn_pmd()
853 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
866 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,  in insert_pfn_pud()  argument
882 			if (pudp_set_access_flags(vma, addr, pud, entry, 1))  in insert_pfn_pud()
883 				update_mmu_cache_pud(vma, addr, pud);  in insert_pfn_pud()
895 	set_pud_at(mm, addr, pud, entry);  in insert_pfn_pud()
896 	update_mmu_cache_pud(vma, addr, pud);  in insert_pfn_pud()
904 	unsigned long addr = vmf->address & PUD_MASK;  in vmf_insert_pfn_pud()  local
919 	if (addr < vma->vm_start || addr >= vma->vm_end)  in vmf_insert_pfn_pud()
924 	insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);  in vmf_insert_pfn_pud()
930 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,  in touch_pmd()  argument
938 	if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,  in touch_pmd()
940 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
943 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,  in follow_devmap_pmd()  argument
967 		touch_pmd(vma, addr, pmd, flags);  in follow_devmap_pmd()
976 	pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;  in follow_devmap_pmd()
987 		  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,  in copy_huge_pmd()  argument
1021 			set_pmd_at(src_mm, addr, src_pmd, pmd);  in copy_huge_pmd()
1026 		set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1049 		set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,  in copy_huge_pmd()
1063 	pmdp_set_wrprotect(src_mm, addr, src_pmd);  in copy_huge_pmd()
1065 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1076 static void touch_pud(struct vm_area_struct *vma, unsigned long addr,  in touch_pud()  argument
1084 	if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,  in touch_pud()
1086 		update_mmu_cache_pud(vma, addr, pud);  in touch_pud()
1089 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,  in follow_devmap_pud()  argument
1107 		touch_pud(vma, addr, pud, flags);  in follow_devmap_pud()
1116 	pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;  in follow_devmap_pud()
1127 		  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,  in copy_huge_pud()  argument
1152 	pudp_set_wrprotect(src_mm, addr, src_pud);  in copy_huge_pud()
1154 	set_pud_at(dst_mm, addr, dst_pud, pud);  in copy_huge_pud()
1475 				   unsigned long addr,  in follow_trans_huge_pmd()  argument
1498 		touch_pmd(vma, addr, pmd, flags);  in follow_trans_huge_pmd()
1533 	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;  in follow_trans_huge_pmd()
1704 		pmd_t *pmd, unsigned long addr, unsigned long next)  in madvise_free_huge_pmd()  argument
1743 	if (next - addr != HPAGE_PMD_SIZE) {  in madvise_free_huge_pmd()
1757 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1761 		set_pmd_at(mm, addr, pmd, orig_pmd);  in madvise_free_huge_pmd()
1762 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in madvise_free_huge_pmd()
1783 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1799 	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,  in zap_huge_pmd()
1801 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1934 		unsigned long addr, pgprot_t newprot, int prot_numa)  in change_huge_pmd()  argument
1964 			set_pmd_at(mm, addr, pmd, newpmd);  in change_huge_pmd()
2002 	entry = pmdp_invalidate(vma, addr, pmd);  in change_huge_pmd()
2008 	set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
2051 		 pud_t *pud, unsigned long addr)  in zap_huge_pud()  argument
2064 	pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm);  in zap_huge_pud()
2065 	tlb_remove_pud_tlb_entry(tlb, pud, addr);  in zap_huge_pud()
2156 	unsigned long addr;  in __split_huge_pmd_locked()  local
2248 	for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {  in __split_huge_pmd_locked()
2271 		pte = pte_offset_map(&_pmd, addr);  in __split_huge_pmd_locked()
2273 		set_pte_at(mm, addr, pte, entry);  in __split_huge_pmd_locked()