492 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
495 		pmd = pmd_mkwrite(pmd);  in maybe_pmd_mkwrite()
496 	return pmd;  in maybe_pmd_mkwrite()
614 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
615 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
642 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
643 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
700 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
704 	if (!pmd_none(*pmd))  in set_huge_zero_page()
709 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in set_huge_zero_page()
710 	set_pmd_at(mm, haddr, pmd, entry);  in set_huge_zero_page()
744 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
747 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
757 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
778 		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,  in insert_pfn_pmd()  argument
785 	ptl = pmd_lock(mm, pmd);  in insert_pfn_pmd()
786 	if (!pmd_none(*pmd)) {  in insert_pfn_pmd()
788 			if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {  in insert_pfn_pmd()
789 				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));  in insert_pfn_pmd()
792 			entry = pmd_mkyoung(*pmd);  in insert_pfn_pmd()
794 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
795 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
810 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in insert_pfn_pmd()
815 	set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
816 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
853 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
931 		pmd_t *pmd, int flags)  in touch_pmd()  argument
935 	_pmd = pmd_mkyoung(*pmd);  in touch_pmd()
939 				pmd, _pmd, flags & FOLL_WRITE))  in touch_pmd()
940 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
944 		pmd_t *pmd, int flags, struct dev_pagemap **pgmap)  in follow_devmap_pmd()  argument
946 	unsigned long pfn = pmd_pfn(*pmd);  in follow_devmap_pmd()
950 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_devmap_pmd()
958 	if (flags & FOLL_WRITE && !pmd_write(*pmd))  in follow_devmap_pmd()
961 	if (pmd_present(*pmd) && pmd_devmap(*pmd))  in follow_devmap_pmd()
967 		touch_pmd(vma, addr, pmd, flags);  in follow_devmap_pmd()
992 	pmd_t pmd;  in copy_huge_pmd()  local
1009 	pmd = *src_pmd;  in copy_huge_pmd()
1012 	if (unlikely(is_swap_pmd(pmd))) {  in copy_huge_pmd()
1013 		swp_entry_t entry = pmd_to_swp_entry(pmd);  in copy_huge_pmd()
1015 		VM_BUG_ON(!is_pmd_migration_entry(pmd));  in copy_huge_pmd()
1018 			pmd = swp_entry_to_pmd(entry);  in copy_huge_pmd()
1020 				pmd = pmd_swp_mksoft_dirty(pmd);  in copy_huge_pmd()
1021 			set_pmd_at(src_mm, addr, src_pmd, pmd);  in copy_huge_pmd()
1026 		set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1032 	if (unlikely(!pmd_trans_huge(pmd))) {  in copy_huge_pmd()
1041 	if (is_huge_zero_pmd(pmd)) {  in copy_huge_pmd()
1055 	src_page = pmd_page(pmd);  in copy_huge_pmd()
1064 	pmd = pmd_mkold(pmd_wrprotect(pmd));  in copy_huge_pmd()
1065 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1191 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1192 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1199 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1200 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1259 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1260 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page_fallback()
1272 	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1274 	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1294 	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);  in do_huge_pmd_wp_page_fallback()
1333 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1338 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page()
1352 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1363 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))  in do_huge_pmd_wp_page()
1364 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1384 			split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1389 				split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1401 		split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1426 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1435 		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page()
1439 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in do_huge_pmd_wp_page()
1440 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1468 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)  in can_follow_write_pmd()  argument
1470 	return pmd_write(pmd) ||  in can_follow_write_pmd()
1471 	       ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));  in can_follow_write_pmd()
1476 				   pmd_t *pmd,  in follow_trans_huge_pmd()  argument
1482 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_trans_huge_pmd()
1484 	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))  in follow_trans_huge_pmd()
1488 	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))  in follow_trans_huge_pmd()
1492 	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))  in follow_trans_huge_pmd()
1495 	page = pmd_page(*pmd);  in follow_trans_huge_pmd()
1498 		touch_pmd(vma, addr, pmd, flags);  in follow_trans_huge_pmd()
1543 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1556 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1557 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1565 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1566 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1574 	page = pmd_page(pmd);  in do_huge_pmd_numa_page()
1585 	if (!pmd_savedwrite(pmd))  in do_huge_pmd_numa_page()
1620 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1667 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1677 	was_writable = pmd_savedwrite(pmd);  in do_huge_pmd_numa_page()
1678 	pmd = pmd_modify(pmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1679 	pmd = pmd_mkyoung(pmd);  in do_huge_pmd_numa_page()
1681 		pmd = pmd_mkwrite(pmd);  in do_huge_pmd_numa_page()
1682 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1683 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1704 		pmd_t *pmd, unsigned long addr, unsigned long next)  in madvise_free_huge_pmd()  argument
1714 	ptl = pmd_trans_huge_lock(pmd, vma);  in madvise_free_huge_pmd()
1718 	orig_pmd = *pmd;  in madvise_free_huge_pmd()
1757 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1761 		set_pmd_at(mm, addr, pmd, orig_pmd);  in madvise_free_huge_pmd()
1762 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in madvise_free_huge_pmd()
1773 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)  in zap_deposited_table()  argument
1777 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in zap_deposited_table()
1783 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1790 	ptl = __pmd_trans_huge_lock(pmd, vma);  in zap_huge_pmd()
1799 	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,  in zap_huge_pmd()
1801 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1804 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1809 		zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1832 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1836 				zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1862 static pmd_t move_soft_dirty_pmd(pmd_t pmd)  in move_soft_dirty_pmd()  argument
1865 	if (unlikely(is_pmd_migration_entry(pmd)))  in move_soft_dirty_pmd()
1866 		pmd = pmd_swp_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1867 	else if (pmd_present(pmd))  in move_soft_dirty_pmd()
1868 		pmd = pmd_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1870 	return pmd;  in move_soft_dirty_pmd()
1878 	pmd_t pmd;  in move_huge_pmd()  local
1905 		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);  in move_huge_pmd()
1906 		if (pmd_present(pmd))  in move_huge_pmd()
1915 		pmd = move_soft_dirty_pmd(pmd);  in move_huge_pmd()
1916 		set_pmd_at(mm, new_addr, new_pmd, pmd);  in move_huge_pmd()
1933 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in change_huge_pmd()  argument
1942 	ptl = __pmd_trans_huge_lock(pmd, vma);  in change_huge_pmd()
1946 	preserve_write = prot_numa && pmd_write(*pmd);  in change_huge_pmd()
1950 	if (is_swap_pmd(*pmd)) {  in change_huge_pmd()
1951 		swp_entry_t entry = pmd_to_swp_entry(*pmd);  in change_huge_pmd()
1953 		VM_BUG_ON(!is_pmd_migration_entry(*pmd));  in change_huge_pmd()
1962 			if (pmd_swp_soft_dirty(*pmd))  in change_huge_pmd()
1964 			set_pmd_at(mm, addr, pmd, newpmd);  in change_huge_pmd()
1975 	if (prot_numa && is_huge_zero_pmd(*pmd))  in change_huge_pmd()
1978 	if (prot_numa && pmd_protnone(*pmd))  in change_huge_pmd()
2002 	entry = pmdp_invalidate(vma, addr, pmd);  in change_huge_pmd()
2008 	set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
2021 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)  in __pmd_trans_huge_lock()  argument
2024 	ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
2025 	if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||  in __pmd_trans_huge_lock()
2026 			pmd_devmap(*pmd)))  in __pmd_trans_huge_lock()
2115 		unsigned long haddr, pmd_t *pmd)  in __split_huge_zero_page_pmd()  argument
2130 	pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
2132 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_zero_page_pmd()
2145 	pmd_populate(mm, pmd, pgtable);  in __split_huge_zero_page_pmd()
2148 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd_locked()  argument
2162 	VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)  in __split_huge_pmd_locked()
2163 				&& !pmd_devmap(*pmd));  in __split_huge_pmd_locked()
2168 		_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_pmd_locked()
2174 			zap_deposited_table(mm, pmd);  in __split_huge_pmd_locked()
2186 	} else if (is_huge_zero_pmd(*pmd)) {  in __split_huge_pmd_locked()
2196 		return __split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_pmd_locked()
2219 	old_pmd = pmdp_invalidate(vma, haddr, pmd);  in __split_huge_pmd_locked()
2245 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_pmd_locked()
2298 	pmd_populate(mm, pmd, pgtable);  in __split_huge_pmd_locked()
2308 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd()  argument
2318 	ptl = pmd_lock(vma->vm_mm, pmd);  in __split_huge_pmd()
2325 	if (page && page != pmd_page(*pmd))  in __split_huge_pmd()
2328 	if (pmd_trans_huge(*pmd)) {  in __split_huge_pmd()
2329 		page = pmd_page(*pmd);  in __split_huge_pmd()
2332 	} else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))  in __split_huge_pmd()
2334 	__split_huge_pmd_locked(vma, pmd, range.start, freeze);  in __split_huge_pmd()
2359 	pmd_t *pmd;  in split_huge_pmd_address()  local
2373 	pmd = pmd_offset(pud, address);  in split_huge_pmd_address()
2375 	__split_huge_pmd(vma, pmd, address, freeze, page);  in split_huge_pmd_address()
3039 	if (!(pvmw->pmd && !pvmw->pte))  in set_pmd_migration_entry()
3043 	pmdval = *pvmw->pmd;  in set_pmd_migration_entry()
3044 	pmdp_invalidate(vma, address, pvmw->pmd);  in set_pmd_migration_entry()
3051 	set_pmd_at(mm, address, pvmw->pmd, pmdswp);  in set_pmd_migration_entry()
3065 	if (!(pvmw->pmd && !pvmw->pte))  in remove_migration_pmd()
3068 	entry = pmd_to_swp_entry(*pvmw->pmd);  in remove_migration_pmd()
3071 	if (pmd_swp_soft_dirty(*pvmw->pmd))  in remove_migration_pmd()
3081 	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);  in remove_migration_pmd()
3084 	update_mmu_cache_pmd(vma, address, pvmw->pmd);  in remove_migration_pmd()