487 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
490 		pmd = pmd_mkwrite(pmd);  in maybe_pmd_mkwrite()
491 	return pmd;  in maybe_pmd_mkwrite()
609 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
610 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
637 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
638 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
695 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
699 	if (!pmd_none(*pmd))  in set_huge_zero_page()
704 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in set_huge_zero_page()
705 	set_pmd_at(mm, haddr, pmd, entry);  in set_huge_zero_page()
738 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
740 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
752 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
772 		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,  in insert_pfn_pmd()  argument
779 	ptl = pmd_lock(mm, pmd);  in insert_pfn_pmd()
780 	if (!pmd_none(*pmd)) {  in insert_pfn_pmd()
782 			if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {  in insert_pfn_pmd()
783 				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));  in insert_pfn_pmd()
786 			entry = pmd_mkyoung(*pmd);  in insert_pfn_pmd()
788 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
789 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
804 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in insert_pfn_pmd()
809 	set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
810 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
847 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
925 		pmd_t *pmd, int flags)  in touch_pmd()  argument
929 	_pmd = pmd_mkyoung(*pmd);  in touch_pmd()
933 				pmd, _pmd, flags & FOLL_WRITE))  in touch_pmd()
934 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
938 		pmd_t *pmd, int flags, struct dev_pagemap **pgmap)  in follow_devmap_pmd()  argument
940 	unsigned long pfn = pmd_pfn(*pmd);  in follow_devmap_pmd()
944 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_devmap_pmd()
952 	if (flags & FOLL_WRITE && !pmd_write(*pmd))  in follow_devmap_pmd()
955 	if (pmd_present(*pmd) && pmd_devmap(*pmd))  in follow_devmap_pmd()
961 		touch_pmd(vma, addr, pmd, flags);  in follow_devmap_pmd()
986 	pmd_t pmd;  in copy_huge_pmd()  local
1003 	pmd = *src_pmd;  in copy_huge_pmd()
1006 	if (unlikely(is_swap_pmd(pmd))) {  in copy_huge_pmd()
1007 		swp_entry_t entry = pmd_to_swp_entry(pmd);  in copy_huge_pmd()
1009 		VM_BUG_ON(!is_pmd_migration_entry(pmd));  in copy_huge_pmd()
1012 			pmd = swp_entry_to_pmd(entry);  in copy_huge_pmd()
1014 				pmd = pmd_swp_mksoft_dirty(pmd);  in copy_huge_pmd()
1015 			set_pmd_at(src_mm, addr, src_pmd, pmd);  in copy_huge_pmd()
1020 		set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1026 	if (unlikely(!pmd_trans_huge(pmd))) {  in copy_huge_pmd()
1035 	if (is_huge_zero_pmd(pmd)) {  in copy_huge_pmd()
1049 	src_page = pmd_page(pmd);  in copy_huge_pmd()
1058 	pmd = pmd_mkold(pmd_wrprotect(pmd));  in copy_huge_pmd()
1059 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1185 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1186 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1193 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1194 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1253 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1254 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page_fallback()
1266 	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1268 	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1288 	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);  in do_huge_pmd_wp_page_fallback()
1327 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1332 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page()
1346 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1357 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))  in do_huge_pmd_wp_page()
1358 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1378 			split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1383 				split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1395 		split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1420 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1429 		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page()
1433 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in do_huge_pmd_wp_page()
1434 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1462 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)  in can_follow_write_pmd()  argument
1464 	return pmd_write(pmd) || ((flags & FOLL_COW) && pmd_dirty(pmd));  in can_follow_write_pmd()
1469 				   pmd_t *pmd,  in follow_trans_huge_pmd()  argument
1475 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_trans_huge_pmd()
1477 	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))  in follow_trans_huge_pmd()
1481 	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))  in follow_trans_huge_pmd()
1485 	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))  in follow_trans_huge_pmd()
1488 	page = pmd_page(*pmd);  in follow_trans_huge_pmd()
1491 		touch_pmd(vma, addr, pmd, flags);  in follow_trans_huge_pmd()
1536 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1549 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1550 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1558 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1559 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1567 	page = pmd_page(pmd);  in do_huge_pmd_numa_page()
1578 	if (!pmd_savedwrite(pmd))  in do_huge_pmd_numa_page()
1613 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1660 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1670 	was_writable = pmd_savedwrite(pmd);  in do_huge_pmd_numa_page()
1671 	pmd = pmd_modify(pmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1672 	pmd = pmd_mkyoung(pmd);  in do_huge_pmd_numa_page()
1674 		pmd = pmd_mkwrite(pmd);  in do_huge_pmd_numa_page()
1675 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1676 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1697 		pmd_t *pmd, unsigned long addr, unsigned long next)  in madvise_free_huge_pmd()  argument
1707 	ptl = pmd_trans_huge_lock(pmd, vma);  in madvise_free_huge_pmd()
1711 	orig_pmd = *pmd;  in madvise_free_huge_pmd()
1750 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1754 		set_pmd_at(mm, addr, pmd, orig_pmd);  in madvise_free_huge_pmd()
1755 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in madvise_free_huge_pmd()
1766 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)  in zap_deposited_table()  argument
1770 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in zap_deposited_table()
1776 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1783 	ptl = __pmd_trans_huge_lock(pmd, vma);  in zap_huge_pmd()
1792 	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,  in zap_huge_pmd()
1794 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1797 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1802 		zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1825 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1829 				zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1855 static pmd_t move_soft_dirty_pmd(pmd_t pmd)  in move_soft_dirty_pmd()  argument
1858 	if (unlikely(is_pmd_migration_entry(pmd)))  in move_soft_dirty_pmd()
1859 		pmd = pmd_swp_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1860 	else if (pmd_present(pmd))  in move_soft_dirty_pmd()
1861 		pmd = pmd_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1863 	return pmd;  in move_soft_dirty_pmd()
1871 	pmd_t pmd;  in move_huge_pmd()  local
1898 		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);  in move_huge_pmd()
1899 		if (pmd_present(pmd))  in move_huge_pmd()
1908 		pmd = move_soft_dirty_pmd(pmd);  in move_huge_pmd()
1909 		set_pmd_at(mm, new_addr, new_pmd, pmd);  in move_huge_pmd()
1926 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in change_huge_pmd()  argument
1935 	ptl = __pmd_trans_huge_lock(pmd, vma);  in change_huge_pmd()
1939 	preserve_write = prot_numa && pmd_write(*pmd);  in change_huge_pmd()
1943 	if (is_swap_pmd(*pmd)) {  in change_huge_pmd()
1944 		swp_entry_t entry = pmd_to_swp_entry(*pmd);  in change_huge_pmd()
1946 		VM_BUG_ON(!is_pmd_migration_entry(*pmd));  in change_huge_pmd()
1955 			if (pmd_swp_soft_dirty(*pmd))  in change_huge_pmd()
1957 			set_pmd_at(mm, addr, pmd, newpmd);  in change_huge_pmd()
1968 	if (prot_numa && is_huge_zero_pmd(*pmd))  in change_huge_pmd()
1971 	if (prot_numa && pmd_protnone(*pmd))  in change_huge_pmd()
1995 	entry = pmdp_invalidate(vma, addr, pmd);  in change_huge_pmd()
2001 	set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
2014 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)  in __pmd_trans_huge_lock()  argument
2017 	ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
2018 	if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||  in __pmd_trans_huge_lock()
2019 			pmd_devmap(*pmd)))  in __pmd_trans_huge_lock()
2108 		unsigned long haddr, pmd_t *pmd)  in __split_huge_zero_page_pmd()  argument
2123 	pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
2125 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_zero_page_pmd()
2138 	pmd_populate(mm, pmd, pgtable);  in __split_huge_zero_page_pmd()
2141 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd_locked()  argument
2155 	VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)  in __split_huge_pmd_locked()
2156 				&& !pmd_devmap(*pmd));  in __split_huge_pmd_locked()
2161 		old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_pmd_locked()
2167 			zap_deposited_table(mm, pmd);  in __split_huge_pmd_locked()
2188 	if (is_huge_zero_pmd(*pmd)) {  in __split_huge_pmd_locked()
2198 		return __split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_pmd_locked()
2221 	old_pmd = pmdp_invalidate(vma, haddr, pmd);  in __split_huge_pmd_locked()
2247 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_pmd_locked()
2306 	pmd_populate(mm, pmd, pgtable);  in __split_huge_pmd_locked()
2316 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd()  argument
2328 	ptl = pmd_lock(vma->vm_mm, pmd);  in __split_huge_pmd()
2337 		if (page != pmd_page(*pmd))  in __split_huge_pmd()
2342 	if (pmd_trans_huge(*pmd)) {  in __split_huge_pmd()
2344 			page = pmd_page(*pmd);  in __split_huge_pmd()
2355 					_pmd = *pmd;  in __split_huge_pmd()
2359 					if (unlikely(!pmd_same(*pmd, _pmd))) {  in __split_huge_pmd()
2372 	} else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))  in __split_huge_pmd()
2374 	__split_huge_pmd_locked(vma, pmd, range.start, freeze);  in __split_huge_pmd()
2401 	pmd_t *pmd;  in split_huge_pmd_address()  local
2415 	pmd = pmd_offset(pud, address);  in split_huge_pmd_address()
2417 	__split_huge_pmd(vma, pmd, address, freeze, page);  in split_huge_pmd_address()
3074 	if (!(pvmw->pmd && !pvmw->pte))  in set_pmd_migration_entry()
3078 	pmdval = pmdp_invalidate(vma, address, pvmw->pmd);  in set_pmd_migration_entry()
3085 	set_pmd_at(mm, address, pvmw->pmd, pmdswp);  in set_pmd_migration_entry()
3099 	if (!(pvmw->pmd && !pvmw->pte))  in remove_migration_pmd()
3102 	entry = pmd_to_swp_entry(*pvmw->pmd);  in remove_migration_pmd()
3105 	if (pmd_swp_soft_dirty(*pvmw->pmd))  in remove_migration_pmd()
3115 	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);  in remove_migration_pmd()
3118 	update_mmu_cache_pmd(vma, address, pvmw->pmd);  in remove_migration_pmd()