Lines Matching full:pmd
551 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) in maybe_pmd_mkwrite() argument
554 pmd = pmd_mkwrite(pmd, vma); in maybe_pmd_mkwrite()
555 return pmd; in maybe_pmd_mkwrite()
679 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
680 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
703 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
704 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in __do_huge_pmd_anonymous_page()
705 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in __do_huge_pmd_anonymous_page()
761 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, in set_huge_zero_page() argument
765 if (!pmd_none(*pmd)) in set_huge_zero_page()
769 pgtable_trans_huge_deposit(mm, pmd, pgtable); in set_huge_zero_page()
770 set_pmd_at(mm, haddr, pmd, entry); in set_huge_zero_page()
802 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
804 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
816 haddr, vmf->pmd, zero_page); in do_huge_pmd_anonymous_page()
817 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_anonymous_page()
836 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, in insert_pfn_pmd() argument
843 ptl = pmd_lock(mm, pmd); in insert_pfn_pmd()
844 if (!pmd_none(*pmd)) { in insert_pfn_pmd()
846 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { in insert_pfn_pmd()
847 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); in insert_pfn_pmd()
850 entry = pmd_mkyoung(*pmd); in insert_pfn_pmd()
852 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) in insert_pfn_pmd()
853 update_mmu_cache_pmd(vma, addr, pmd); in insert_pfn_pmd()
868 pgtable_trans_huge_deposit(mm, pmd, pgtable); in insert_pfn_pmd()
873 set_pmd_at(mm, addr, pmd, entry); in insert_pfn_pmd()
874 update_mmu_cache_pmd(vma, addr, pmd); in insert_pfn_pmd()
883 * vmf_insert_pfn_pmd - insert a pmd size pfn
888 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
921 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); in vmf_insert_pfn_pmd()
1010 pmd_t *pmd, bool write) in touch_pmd() argument
1014 _pmd = pmd_mkyoung(*pmd); in touch_pmd()
1018 pmd, _pmd, write)) in touch_pmd()
1019 update_mmu_cache_pmd(vma, addr, pmd); in touch_pmd()
1023 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) in follow_devmap_pmd() argument
1025 unsigned long pfn = pmd_pfn(*pmd); in follow_devmap_pmd()
1030 assert_spin_locked(pmd_lockptr(mm, pmd)); in follow_devmap_pmd()
1032 if (flags & FOLL_WRITE && !pmd_write(*pmd)) in follow_devmap_pmd()
1035 if (pmd_present(*pmd) && pmd_devmap(*pmd)) in follow_devmap_pmd()
1041 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); in follow_devmap_pmd()
1068 pmd_t pmd; in copy_huge_pmd() local
1085 pmd = *src_pmd; in copy_huge_pmd()
1088 if (unlikely(is_swap_pmd(pmd))) { in copy_huge_pmd()
1089 swp_entry_t entry = pmd_to_swp_entry(pmd); in copy_huge_pmd()
1091 VM_BUG_ON(!is_pmd_migration_entry(pmd)); in copy_huge_pmd()
1095 pmd = swp_entry_to_pmd(entry); in copy_huge_pmd()
1097 pmd = pmd_swp_mksoft_dirty(pmd); in copy_huge_pmd()
1099 pmd = pmd_swp_mkuffd_wp(pmd); in copy_huge_pmd()
1100 set_pmd_at(src_mm, addr, src_pmd, pmd); in copy_huge_pmd()
1106 pmd = pmd_swp_clear_uffd_wp(pmd); in copy_huge_pmd()
1107 set_pmd_at(dst_mm, addr, dst_pmd, pmd); in copy_huge_pmd()
1113 if (unlikely(!pmd_trans_huge(pmd))) { in copy_huge_pmd()
1118 * When page table lock is held, the huge zero pmd should not be in copy_huge_pmd()
1119 * under splitting since we don't split the page itself, only pmd to in copy_huge_pmd()
1122 if (is_huge_zero_pmd(pmd)) { in copy_huge_pmd()
1132 src_page = pmd_page(pmd); in copy_huge_pmd()
1151 pmd = pmd_clear_uffd_wp(pmd); in copy_huge_pmd()
1152 pmd = pmd_mkold(pmd_wrprotect(pmd)); in copy_huge_pmd()
1153 set_pmd_at(dst_mm, addr, dst_pmd, pmd); in copy_huge_pmd()
1279 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1280 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) in huge_pmd_set_accessed()
1283 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); in huge_pmd_set_accessed()
1298 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1306 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1324 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1361 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1362 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1371 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in do_huge_pmd_wp_page()
1376 unsigned long addr, pmd_t pmd) in can_change_pmd_writable() argument
1384 if (pmd_protnone(pmd)) in can_change_pmd_writable()
1388 if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd)) in can_change_pmd_writable()
1392 if (userfaultfd_huge_pmd_wp(vma, pmd)) in can_change_pmd_writable()
1397 page = vm_normal_page_pmd(vma, addr, pmd); in can_change_pmd_writable()
1402 return pmd_dirty(pmd); in can_change_pmd_writable()
1406 static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page, in can_follow_write_pmd() argument
1410 /* If the pmd is writable, we can write to the page. */ in can_follow_write_pmd()
1411 if (pmd_write(pmd)) in can_follow_write_pmd()
1438 if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd)) in can_follow_write_pmd()
1440 return !userfaultfd_huge_pmd_wp(vma, pmd); in can_follow_write_pmd()
1445 pmd_t *pmd, in follow_trans_huge_pmd() argument
1452 assert_spin_locked(pmd_lockptr(mm, pmd)); in follow_trans_huge_pmd()
1454 page = pmd_page(*pmd); in follow_trans_huge_pmd()
1458 !can_follow_write_pmd(*pmd, page, vma, flags)) in follow_trans_huge_pmd()
1462 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) in follow_trans_huge_pmd()
1465 if (pmd_protnone(*pmd) && !gup_can_follow_protnone(vma, flags)) in follow_trans_huge_pmd()
1468 if (!pmd_write(*pmd) && gup_must_unshare(vma, flags, page)) in follow_trans_huge_pmd()
1479 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); in follow_trans_huge_pmd()
1492 pmd_t pmd; in do_huge_pmd_numa_page() local
1500 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1501 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1506 pmd = pmd_modify(oldpmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1509 * Detect now whether the PMD could be writable; this information in do_huge_pmd_numa_page()
1512 writable = pmd_write(pmd); in do_huge_pmd_numa_page()
1514 can_change_pmd_writable(vma, vmf->address, pmd)) in do_huge_pmd_numa_page()
1517 page = vm_normal_page_pmd(vma, haddr, pmd); in do_huge_pmd_numa_page()
1549 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1550 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1565 /* Restore the PMD */ in do_huge_pmd_numa_page()
1566 pmd = pmd_modify(oldpmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1567 pmd = pmd_mkyoung(pmd); in do_huge_pmd_numa_page()
1569 pmd = pmd_mkwrite(pmd, vma); in do_huge_pmd_numa_page()
1570 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
1571 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
1577 * Return true if we do MADV_FREE successfully on entire pmd page.
1581 pmd_t *pmd, unsigned long addr, unsigned long next) in madvise_free_huge_pmd() argument
1591 ptl = pmd_trans_huge_lock(pmd, vma); in madvise_free_huge_pmd()
1595 orig_pmd = *pmd; in madvise_free_huge_pmd()
1634 pmdp_invalidate(vma, addr, pmd); in madvise_free_huge_pmd()
1638 set_pmd_at(mm, addr, pmd, orig_pmd); in madvise_free_huge_pmd()
1639 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_free_huge_pmd()
1650 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) in zap_deposited_table() argument
1654 pgtable = pgtable_trans_huge_withdraw(mm, pmd); in zap_deposited_table()
1660 pmd_t *pmd, unsigned long addr) in zap_huge_pmd() argument
1667 ptl = __pmd_trans_huge_lock(pmd, vma); in zap_huge_pmd()
1676 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, in zap_huge_pmd()
1679 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1682 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1685 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1704 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); in zap_huge_pmd()
1707 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1711 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1728 * With split pmd lock we also need to move preallocated in pmd_move_must_withdraw()
1729 * PTE page table if new_pmd is on different PMD page table. in pmd_move_must_withdraw()
1737 static pmd_t move_soft_dirty_pmd(pmd_t pmd) in move_soft_dirty_pmd() argument
1740 if (unlikely(is_pmd_migration_entry(pmd))) in move_soft_dirty_pmd()
1741 pmd = pmd_swp_mksoft_dirty(pmd); in move_soft_dirty_pmd()
1742 else if (pmd_present(pmd)) in move_soft_dirty_pmd()
1743 pmd = pmd_mksoft_dirty(pmd); in move_soft_dirty_pmd()
1745 return pmd; in move_soft_dirty_pmd()
1752 pmd_t pmd; in move_huge_pmd() local
1757 * The destination pmd shouldn't be established, free_pgtables() in move_huge_pmd()
1775 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); in move_huge_pmd()
1776 if (pmd_present(pmd)) in move_huge_pmd()
1785 pmd = move_soft_dirty_pmd(pmd); in move_huge_pmd()
1786 set_pmd_at(mm, new_addr, new_pmd, pmd); in move_huge_pmd()
1799 * - 0 if PMD could not be locked
1800 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
1805 pmd_t *pmd, unsigned long addr, pgprot_t newprot, in change_huge_pmd() argument
1821 ptl = __pmd_trans_huge_lock(pmd, vma); in change_huge_pmd()
1826 if (is_swap_pmd(*pmd)) { in change_huge_pmd()
1827 swp_entry_t entry = pmd_to_swp_entry(*pmd); in change_huge_pmd()
1831 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); in change_huge_pmd()
1842 if (pmd_swp_soft_dirty(*pmd)) in change_huge_pmd()
1845 newpmd = *pmd; in change_huge_pmd()
1852 if (!pmd_same(*pmd, newpmd)) in change_huge_pmd()
1853 set_pmd_at(mm, addr, pmd, newpmd); in change_huge_pmd()
1866 if (is_huge_zero_pmd(*pmd)) in change_huge_pmd()
1869 if (pmd_protnone(*pmd)) in change_huge_pmd()
1872 page = pmd_page(*pmd); in change_huge_pmd()
1888 * to not clear pmd intermittently to avoid race with MADV_DONTNEED in change_huge_pmd()
1896 * pmd_trans_huge(*pmd) == 0 (without ptl) in change_huge_pmd()
1897 * // skip the pmd in change_huge_pmd()
1899 * // pmd is re-established in change_huge_pmd()
1901 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it in change_huge_pmd()
1907 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); in change_huge_pmd()
1926 set_pmd_at(mm, addr, pmd, entry); in change_huge_pmd()
1936 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1941 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) in __pmd_trans_huge_lock() argument
1944 ptl = pmd_lock(vma->vm_mm, pmd); in __pmd_trans_huge_lock()
1945 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || in __pmd_trans_huge_lock()
1946 pmd_devmap(*pmd))) in __pmd_trans_huge_lock()
2026 unsigned long haddr, pmd_t *pmd) in __split_huge_zero_page_pmd() argument
2036 * Leave pmd empty until pte is filled note that it is fine to delay in __split_huge_zero_page_pmd()
2038 * replacing a zero pmd write protected page with a zero pte write in __split_huge_zero_page_pmd()
2043 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); in __split_huge_zero_page_pmd()
2045 pgtable = pgtable_trans_huge_withdraw(mm, pmd); in __split_huge_zero_page_pmd()
2062 smp_wmb(); /* make pte visible before pmd */ in __split_huge_zero_page_pmd()
2063 pmd_populate(mm, pmd, pgtable); in __split_huge_zero_page_pmd()
2066 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, in __split_huge_pmd_locked() argument
2082 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) in __split_huge_pmd_locked()
2083 && !pmd_devmap(*pmd)); in __split_huge_pmd_locked()
2088 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); in __split_huge_pmd_locked()
2094 zap_deposited_table(mm, pmd); in __split_huge_pmd_locked()
2115 if (is_huge_zero_pmd(*pmd)) { in __split_huge_pmd_locked()
2125 return __split_huge_zero_page_pmd(vma, haddr, pmd); in __split_huge_pmd_locked()
2129 * Up to this point the pmd is present and huge and userland has the in __split_huge_pmd_locked()
2131 * place). If we overwrite the pmd with the not-huge version pointing in __split_huge_pmd_locked()
2143 * current pmd notpresent (atomically because here the pmd_trans_huge in __split_huge_pmd_locked()
2144 * must remain set at all times on the pmd until the split is complete in __split_huge_pmd_locked()
2145 * for this pmd), then we flush the SMP TLB and finally we write the in __split_huge_pmd_locked()
2146 * non-huge version of the pmd entry with pmd_populate. in __split_huge_pmd_locked()
2148 old_pmd = pmdp_invalidate(vma, haddr, pmd); in __split_huge_pmd_locked()
2177 * Without "freeze", we'll simply split the PMD, propagating the in __split_huge_pmd_locked()
2186 * In case we cannot clear PageAnonExclusive(), split the PMD in __split_huge_pmd_locked()
2189 * See page_try_share_anon_rmap(): invalidate PMD first. in __split_huge_pmd_locked()
2199 * Withdraw the table only after we mark the pmd entry invalid. in __split_huge_pmd_locked()
2202 pgtable = pgtable_trans_huge_withdraw(mm, pmd); in __split_huge_pmd_locked()
2262 smp_wmb(); /* make pte visible before pmd */ in __split_huge_pmd_locked()
2263 pmd_populate(mm, pmd, pgtable); in __split_huge_pmd_locked()
2266 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, in __split_huge_pmd() argument
2276 ptl = pmd_lock(vma->vm_mm, pmd); in __split_huge_pmd()
2280 * pmd against. Otherwise we can end up replacing wrong folio. in __split_huge_pmd()
2285 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) || in __split_huge_pmd()
2286 is_pmd_migration_entry(*pmd)) { in __split_huge_pmd()
2289 * guaranteed that pmd is present. in __split_huge_pmd()
2291 if (folio && folio != page_folio(pmd_page(*pmd))) in __split_huge_pmd()
2293 __split_huge_pmd_locked(vma, pmd, range.start, freeze); in __split_huge_pmd()
2304 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); in split_huge_pmd_address() local
2306 if (!pmd) in split_huge_pmd_address()
2309 __split_huge_pmd(vma, pmd, address, freeze, folio); in split_huge_pmd_address()
2316 * contain an hugepage: check if we need to split an huge pmd. in split_huge_pmd_if_needed()
3215 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
3219 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
3221 /* See page_try_share_anon_rmap(): invalidate PMD first. */ in set_pmd_migration_entry()
3224 set_pmd_at(mm, address, pvmw->pmd, pmdval); in set_pmd_migration_entry()
3245 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
3262 if (!(pvmw->pmd && !pvmw->pte)) in remove_migration_pmd()
3265 entry = pmd_to_swp_entry(*pvmw->pmd); in remove_migration_pmd()
3268 if (pmd_swp_soft_dirty(*pvmw->pmd)) in remove_migration_pmd()
3272 if (pmd_swp_uffd_wp(*pvmw->pmd)) in remove_migration_pmd()
3291 set_pmd_at(mm, haddr, pvmw->pmd, pmde); in remove_migration_pmd()
3294 update_mmu_cache_pmd(vma, address, pvmw->pmd); in remove_migration_pmd()