Lines Matching +full:tlb +full:- +full:split
1 // SPDX-License-Identifier: GPL-2.0-only
21 #include <linux/backing-dev.h>
39 #include <linux/memory-tiers.h>
41 #include <asm/tlb.h>
77 if (!vma->vm_mm) /* vdso */ in hugepage_vma_check()
86 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) in hugepage_vma_check()
113 !transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) in hugepage_vma_check()
121 if (!in_pf && shmem_file(vma->vm_file)) in hugepage_vma_check()
122 return shmem_is_huge(file_inode(vma->vm_file), vma->vm_pgoff, in hugepage_vma_check()
123 !enforce_sysfs, vma->vm_mm, vm_flags); in hugepage_vma_check()
148 if (!vma->anon_vma) in hugepage_vma_check()
193 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
199 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
207 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_put_huge_zero_page()
271 ret = -EINVAL; in enabled_store()
303 return -EINVAL; in single_hugepage_flag_store()
366 return -EINVAL; in defrag_store()
416 return -ENOMEM; in hugepage_init_sysfs()
464 return -EINVAL; in hugepage_init()
472 * we use page->mapping and page->index in second tail page in hugepage_init()
485 err = register_shrinker(&huge_zero_page_shrinker, "thp-zero"); in hugepage_init()
488 err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split"); in hugepage_init()
494 * where the extra memory used could hurt more than TLB overhead in hugepage_init()
497 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { in hugepage_init()
553 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pmd_mkwrite()
566 return &memcg->deferred_split_queue; in get_deferred_split_queue()
568 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
576 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
583 INIT_LIST_HEAD(&folio->_deferred_list); in folio_prep_large_rmappable()
592 return is_huge_zero_page(&folio->page) || in is_transparent_hugepage()
604 if (off_end <= off_align || (off_end - off_align) < size) in __thp_get_unmapped_area()
611 ret = current->mm->get_unmapped_area(filp, addr, len_pad, in __thp_get_unmapped_area()
628 ret += (off - ret) & (size - 1); in __thp_get_unmapped_area()
642 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); in thp_get_unmapped_area()
649 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
652 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
657 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { in __do_huge_pmd_anonymous_page()
665 pgtable = pte_alloc_one(vma->vm_mm); in __do_huge_pmd_anonymous_page()
671 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
679 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
680 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
685 ret = check_stable_address_space(vma->vm_mm); in __do_huge_pmd_anonymous_page()
691 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
693 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
699 entry = mk_huge_pmd(page, vma->vm_page_prot); in __do_huge_pmd_anonymous_page()
703 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
704 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in __do_huge_pmd_anonymous_page()
705 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in __do_huge_pmd_anonymous_page()
706 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
707 mm_inc_nr_ptes(vma->vm_mm); in __do_huge_pmd_anonymous_page()
708 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
710 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); in __do_huge_pmd_anonymous_page()
715 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
718 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
735 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); in vma_thp_gfp_mask()
767 entry = mk_pmd(zero_page, vma->vm_page_prot); in set_huge_zero_page()
776 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_anonymous_page()
779 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_anonymous_page()
785 khugepaged_enter_vma(vma, vma->vm_flags); in do_huge_pmd_anonymous_page()
787 if (!(vmf->flags & FAULT_FLAG_WRITE) && in do_huge_pmd_anonymous_page()
788 !mm_forbids_zeropage(vma->vm_mm) && in do_huge_pmd_anonymous_page()
793 pgtable = pte_alloc_one(vma->vm_mm); in do_huge_pmd_anonymous_page()
796 zero_page = mm_get_huge_zero_page(vma->vm_mm); in do_huge_pmd_anonymous_page()
798 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
802 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
804 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
805 ret = check_stable_address_space(vma->vm_mm); in do_huge_pmd_anonymous_page()
807 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
808 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
810 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
811 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
815 set_huge_zero_page(pgtable, vma->vm_mm, vma, in do_huge_pmd_anonymous_page()
816 haddr, vmf->pmd, zero_page); in do_huge_pmd_anonymous_page()
817 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_anonymous_page()
818 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
821 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
822 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
832 return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp); in do_huge_pmd_anonymous_page()
839 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pmd()
883 * vmf_insert_pfn_pmd - insert a pmd size pfn
894 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_pfn_pmd()
895 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pmd()
896 pgprot_t pgprot = vma->vm_page_prot; in vmf_insert_pfn_pmd()
904 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pmd()
906 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pmd()
908 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pmd()
910 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pmd()
914 pgtable = pte_alloc_one(vma->vm_mm); in vmf_insert_pfn_pmd()
921 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); in vmf_insert_pfn_pmd()
929 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pud_mkwrite()
937 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pud()
938 pgprot_t prot = vma->vm_page_prot; in insert_pfn_pud()
972 * vmf_insert_pfn_pud - insert a pud size pfn
983 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_pfn_pud()
984 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pud()
985 pgprot_t pgprot = vma->vm_page_prot; in vmf_insert_pfn_pud()
992 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pud()
994 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pud()
996 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pud()
998 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pud()
1003 insert_pfn_pud(vma, addr, vmf->pud, pfn, write); in vmf_insert_pfn_pud()
1026 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pmd()
1048 return ERR_PTR(-EEXIST); in follow_devmap_pmd()
1053 return ERR_PTR(-EFAULT); in follow_devmap_pmd()
1070 int ret = -ENOMEM; in copy_huge_pmd()
1072 /* Skip if can be re-fill on fault */ in copy_huge_pmd()
1084 ret = -EAGAIN; in copy_huge_pmd()
1119 * under splitting since we don't split the page itself, only pmd to in copy_huge_pmd()
1137 /* Page maybe pinned: split and retry the fault on PTEs. */ in copy_huge_pmd()
1143 return -EAGAIN; in copy_huge_pmd()
1181 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pud()
1205 return ERR_PTR(-EEXIST); in follow_devmap_pud()
1210 return ERR_PTR(-EFAULT); in follow_devmap_pud()
1232 ret = -EAGAIN; in copy_huge_pud()
1239 * under splitting since we don't split the page itself, only pud to in copy_huge_pud()
1248 * and split if duplicating fails. in copy_huge_pud()
1263 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pud_set_accessed()
1265 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); in huge_pud_set_accessed()
1266 if (unlikely(!pud_same(*vmf->pud, orig_pud))) in huge_pud_set_accessed()
1269 touch_pud(vmf->vma, vmf->address, vmf->pud, write); in huge_pud_set_accessed()
1271 spin_unlock(vmf->ptl); in huge_pud_set_accessed()
1277 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pmd_set_accessed()
1279 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1280 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) in huge_pmd_set_accessed()
1283 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); in huge_pmd_set_accessed()
1286 spin_unlock(vmf->ptl); in huge_pmd_set_accessed()
1291 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; in do_huge_pmd_wp_page()
1292 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_wp_page()
1295 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_wp_page()
1296 pmd_t orig_pmd = vmf->orig_pmd; in do_huge_pmd_wp_page()
1298 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1299 VM_BUG_ON_VMA(!vma->anon_vma, vma); in do_huge_pmd_wp_page()
1304 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1306 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1307 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1321 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1323 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1324 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1325 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1356 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1361 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1362 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1363 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1369 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1371 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in do_huge_pmd_wp_page()
1380 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) in can_change_pmd_writable()
1391 /* Do we need write faults for uffd-wp tracking? */ in can_change_pmd_writable()
1395 if (!(vma->vm_flags & VM_SHARED)) { in can_change_pmd_writable()
1419 if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) in can_follow_write_pmd()
1422 /* ... or read-only private ones */ in can_follow_write_pmd()
1423 if (!(vma->vm_flags & VM_MAYWRITE)) in can_follow_write_pmd()
1427 if (vma->vm_flags & VM_WRITE) in can_follow_write_pmd()
1437 /* ... and a write-fault isn't required for other reasons. */ in can_follow_write_pmd()
1448 struct mm_struct *mm = vma->vm_mm; in follow_trans_huge_pmd()
1463 return ERR_PTR(-EFAULT); in follow_trans_huge_pmd()
1469 return ERR_PTR(-EMLINK); in follow_trans_huge_pmd()
1490 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_numa_page()
1491 pmd_t oldpmd = vmf->orig_pmd; in do_huge_pmd_numa_page()
1494 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_numa_page()
1496 int target_nid, last_cpupid = (-1 & LAST_CPUPID_MASK); in do_huge_pmd_numa_page()
1500 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1501 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1502 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1506 pmd = pmd_modify(oldpmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1514 can_change_pmd_writable(vma, vmf->address, pmd)) in do_huge_pmd_numa_page()
1540 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1549 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1550 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1551 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1566 pmd = pmd_modify(oldpmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1570 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
1571 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
1572 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1580 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in madvise_free_huge_pmd() argument
1586 struct mm_struct *mm = tlb->mm; in madvise_free_huge_pmd()
1589 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_free_huge_pmd()
1617 * If user want to discard part-pages of THP, split it so MADV_FREE in madvise_free_huge_pmd()
1620 if (next - addr != HPAGE_PMD_SIZE) { in madvise_free_huge_pmd()
1639 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_free_huge_pmd()
1659 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pmd() argument
1665 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in zap_huge_pmd()
1677 tlb->fullmm); in zap_huge_pmd()
1679 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1682 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1685 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1707 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1708 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); in zap_huge_pmd()
1711 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1712 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); in zap_huge_pmd()
1717 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); in zap_huge_pmd()
1728 * With split pmd lock we also need to move preallocated in pmd_move_must_withdraw()
1753 struct mm_struct *mm = vma->vm_mm; in move_huge_pmd()
1799 * - 0 if PMD could not be locked
1800 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
1802 * - HPAGE_PMD_NR if protections changed and TLB flush necessary
1804 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in change_huge_pmd() argument
1808 struct mm_struct *mm = vma->vm_mm; in change_huge_pmd()
1816 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in change_huge_pmd()
1862 * Avoid trapping faults against the zero page. The read-only in change_huge_pmd()
1863 * data is likely to be read-cached on the local CPU and in change_huge_pmd()
1899 * // pmd is re-established in change_huge_pmd()
1929 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); in change_huge_pmd()
1944 ptl = pmd_lock(vma->vm_mm, pmd); in __pmd_trans_huge_lock()
1962 ptl = pud_lock(vma->vm_mm, pud); in __pud_trans_huge_lock()
1970 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pud() argument
1979 pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); in zap_huge_pud()
1980 tlb_remove_pud_tlb_entry(tlb, pud, addr); in zap_huge_pud()
1995 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pud_locked()
1996 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); in __split_huge_pud_locked()
2010 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in __split_huge_pud()
2014 ptl = pud_lock(vma->vm_mm, pud); in __split_huge_pud()
2028 struct mm_struct *mm = vma->vm_mm; in __split_huge_zero_page_pmd()
2053 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); in __split_huge_zero_page_pmd()
2061 pte_unmap(pte - 1); in __split_huge_zero_page_pmd()
2069 struct mm_struct *mm = vma->vm_mm; in __split_huge_pmd_locked()
2080 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pmd_locked()
2081 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); in __split_huge_pmd_locked()
2111 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); in __split_huge_pmd_locked()
2130 * whole access to the hugepage during the split (which happens in in __split_huge_pmd_locked()
2131 * place). If we overwrite the pmd with the not-huge version pointing in __split_huge_pmd_locked()
2133 * free), userland could trigger a small page size TLB miss on the in __split_huge_pmd_locked()
2134 * small sized TLB while the hugepage TLB entry is still established in in __split_huge_pmd_locked()
2135 * the huge TLB. Some CPU doesn't like that. in __split_huge_pmd_locked()
2139 * loaded in the two TLB is identical (which should be the case here). in __split_huge_pmd_locked()
2140 * But it is generally safer to never allow small and huge TLB entries in __split_huge_pmd_locked()
2144 * must remain set at all times on the pmd until the split is complete in __split_huge_pmd_locked()
2145 * for this pmd), then we flush the SMP TLB and finally we write the in __split_huge_pmd_locked()
2146 * non-huge version of the pmd entry with pmd_populate. in __split_huge_pmd_locked()
2177 * Without "freeze", we'll simply split the PMD, propagating the in __split_huge_pmd_locked()
2179 * each subpage -- no need to (temporarily) clear. in __split_huge_pmd_locked()
2183 * managed to clear PageAnonExclusive() -- see in __split_huge_pmd_locked()
2186 * In case we cannot clear PageAnonExclusive(), split the PMD in __split_huge_pmd_locked()
2195 page_ref_add(page, HPAGE_PMD_NR - 1); in __split_huge_pmd_locked()
2235 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); in __split_huge_pmd_locked()
2242 /* NOTE: this may set soft-dirty too on some archs */ in __split_huge_pmd_locked()
2255 pte_unmap(pte - 1); in __split_huge_pmd_locked()
2272 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in __split_huge_pmd()
2276 ptl = pmd_lock(vma->vm_mm, pmd); in __split_huge_pmd()
2304 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); in split_huge_pmd_address()
2316 * contain an hugepage: check if we need to split an huge pmd. in split_huge_pmd_if_needed()
2329 /* Check if we need to split start first. */ in vma_adjust_trans_huge()
2332 /* Check if we need to split end next. */ in vma_adjust_trans_huge()
2337 * check if we need to split it. in vma_adjust_trans_huge()
2340 struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); in vma_adjust_trans_huge()
2341 unsigned long nstart = next->vm_start; in vma_adjust_trans_huge()
2387 lockdep_assert_held(&lruvec->lru_lock); in lru_add_page_tail()
2393 list_add_tail(&tail->lru, list); in lru_add_page_tail()
2398 tail->mlock_count = 0; in lru_add_page_tail()
2400 list_add_tail(&tail->lru, &head->lru); in lru_add_page_tail()
2408 struct page *head = &folio->page; in __split_huge_page_tail()
2416 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); in __split_huge_page_tail()
2424 * Note that for mapped sub-pages of an anonymous THP, in __split_huge_page_tail()
2428 * unreferenced sub-pages of an anonymous THP: we can simply drop in __split_huge_page_tail()
2429 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. in __split_huge_page_tail()
2431 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; in __split_huge_page_tail()
2432 page_tail->flags |= (head->flags & in __split_huge_page_tail()
2449 /* ->mapping in first and second tail page is replaced by other uses */ in __split_huge_page_tail()
2450 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, in __split_huge_page_tail()
2452 page_tail->mapping = head->mapping; in __split_huge_page_tail()
2453 page_tail->index = head->index + tail; in __split_huge_page_tail()
2456 * page->private should not be set in tail pages. Fix up and warn once in __split_huge_page_tail()
2459 if (unlikely(page_tail->private)) { in __split_huge_page_tail()
2461 page_tail->private = 0; in __split_huge_page_tail()
2464 new_folio->swap.val = folio->swap.val + tail; in __split_huge_page_tail()
2466 /* Page flags must be visible before we make the page non-compound. */ in __split_huge_page_tail()
2490 * pages to show after the currently processed elements - e.g. in __split_huge_page_tail()
2500 struct page *head = &folio->page; in __split_huge_page()
2511 offset = swp_offset(folio->swap); in __split_huge_page()
2512 swap_cache = swap_address_space(folio->swap); in __split_huge_page()
2513 xa_lock(&swap_cache->i_pages); in __split_huge_page()
2521 for (i = nr - 1; i >= 1; i--) { in __split_huge_page()
2527 if (shmem_mapping(head->mapping)) in __split_huge_page()
2531 inode_to_wb(folio->mapping->host)); in __split_huge_page()
2535 __xa_store(&head->mapping->i_pages, head[i].index, in __split_huge_page()
2538 __xa_store(&swap_cache->i_pages, offset + i, in __split_huge_page()
2554 xa_unlock(&swap_cache->i_pages); in __split_huge_page()
2561 xa_unlock(&head->mapping->i_pages); in __split_huge_page()
2566 shmem_uncharge(head->mapping->host, nr_dropped); in __split_huge_page()
2570 split_swap_cluster(folio->swap); in __split_huge_page()
2583 * of the tail pages after the split is complete. in __split_huge_page()
2589 /* Racy check whether the huge page can be split */
2602 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 1; in can_split_folio()
2607 * subpage of huge page to split. Split doesn't change the position of @page.
2609 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2620 * Returns 0 if the hugepage is split successfully.
2621 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2628 XA_STATE(xas, &folio->mapping->i_pages, folio->index); in split_huge_page_to_list()
2638 is_hzp = is_huge_zero_page(&folio->page); in split_huge_page_to_list()
2641 return -EBUSY; in split_huge_page_to_list()
2645 return -EBUSY; in split_huge_page_to_list()
2653 * is taken to serialise against parallel split or collapse in split_huge_page_to_list()
2658 ret = -EBUSY; in split_huge_page_to_list()
2661 end = -1; in split_huge_page_to_list()
2667 mapping = folio->mapping; in split_huge_page_to_list()
2671 ret = -EBUSY; in split_huge_page_to_list()
2679 ret = -EBUSY; in split_huge_page_to_list()
2694 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, in split_huge_page_to_list()
2699 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in split_huge_page_to_list()
2701 end = shmem_fallocend(mapping->host, end); in split_huge_page_to_list()
2705 * Racy check if we can split the page, before unmap_folio() will in split_huge_page_to_list()
2706 * split PMDs in split_huge_page_to_list()
2709 ret = -EAGAIN; in split_huge_page_to_list()
2728 /* Prevent deferred_split_scan() touching ->_refcount */ in split_huge_page_to_list()
2729 spin_lock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2731 if (!list_empty(&folio->_deferred_list)) { in split_huge_page_to_list()
2732 ds_queue->split_queue_len--; in split_huge_page_to_list()
2733 list_del(&folio->_deferred_list); in split_huge_page_to_list()
2735 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2743 NR_SHMEM_THPS, -nr); in split_huge_page_to_list()
2746 NR_FILE_THPS, -nr); in split_huge_page_to_list()
2755 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2761 ret = -EAGAIN; in split_huge_page_to_list()
2787 if (data_race(list_empty(&folio->_deferred_list))) in folio_undo_large_rmappable()
2791 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in folio_undo_large_rmappable()
2792 if (!list_empty(&folio->_deferred_list)) { in folio_undo_large_rmappable()
2793 ds_queue->split_queue_len--; in folio_undo_large_rmappable()
2794 list_del(&folio->_deferred_list); in folio_undo_large_rmappable()
2796 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in folio_undo_large_rmappable()
2811 * this may cause a race condition to corrupt deferred split queue. in deferred_split_folio()
2822 if (!list_empty(&folio->_deferred_list)) in deferred_split_folio()
2825 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_folio()
2826 if (list_empty(&folio->_deferred_list)) { in deferred_split_folio()
2828 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); in deferred_split_folio()
2829 ds_queue->split_queue_len++; in deferred_split_folio()
2836 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_folio()
2842 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_count()
2843 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_count()
2846 if (sc->memcg) in deferred_split_count()
2847 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_count()
2849 return READ_ONCE(ds_queue->split_queue_len); in deferred_split_count()
2855 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_scan()
2856 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_scan()
2860 int split = 0; in deferred_split_scan() local
2863 if (sc->memcg) in deferred_split_scan()
2864 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_scan()
2867 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2869 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, in deferred_split_scan()
2872 list_move(&folio->_deferred_list, &list); in deferred_split_scan()
2875 list_del_init(&folio->_deferred_list); in deferred_split_scan()
2876 ds_queue->split_queue_len--; in deferred_split_scan()
2878 if (!--sc->nr_to_scan) in deferred_split_scan()
2881 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2888 split++; in deferred_split_scan()
2894 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2895 list_splice_tail(&list, &ds_queue->split_queue); in deferred_split_scan()
2896 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2899 * Stop shrinker if we didn't split any page, but the queue is empty. in deferred_split_scan()
2902 if (!split && list_empty(&ds_queue->split_queue)) in deferred_split_scan()
2904 return split; in deferred_split_scan()
2922 unsigned long total = 0, split = 0; in split_huge_pages_all() local
2924 pr_debug("Split all THPs\n"); in split_huge_pages_all()
2929 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { in split_huge_pages_all()
2954 split++; in split_huge_pages_all()
2955 pfn += nr_pages - 1; in split_huge_pages_all()
2963 pr_debug("%lu of %lu THP split\n", split, total); in split_huge_pages_all()
2968 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || in vma_not_suitable_for_thp_split()
2978 unsigned long total = 0, split = 0; in split_huge_pages_pid() local
2989 ret = -ESRCH; in split_huge_pages_pid()
3000 ret = -EINVAL; in split_huge_pages_pid()
3004 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", in split_huge_pages_pid()
3010 * table filled with PTE-mapped THPs, each of which is distinct. in split_huge_pages_pid()
3022 addr = vma->vm_end; in split_huge_pages_pid()
3044 split++; in split_huge_pages_pid()
3054 pr_debug("%lu of %lu THP split\n", split, total); in split_huge_pages_pid()
3066 int ret = -EINVAL; in split_huge_pages_in_file()
3069 unsigned long total = 0, split = 0; in split_huge_pages_in_file() local
3079 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", in split_huge_pages_in_file()
3082 mapping = candidate->f_mapping; in split_huge_pages_in_file()
3101 split++; in split_huge_pages_in_file()
3112 pr_debug("%lu of %lu file-backed THP split\n", split, total); in split_huge_pages_in_file()
3134 ret = -EFAULT; in split_huge_pages_write()
3140 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; in split_huge_pages_write()
3153 ret = -EINVAL; in split_huge_pages_write()
3159 ret = -EINVAL; in split_huge_pages_write()
3175 ret = -EINVAL; in split_huge_pages_write()
3207 struct vm_area_struct *vma = pvmw->vma; in set_pmd_migration_entry()
3208 struct mm_struct *mm = vma->vm_mm; in set_pmd_migration_entry()
3209 unsigned long address = pvmw->address; in set_pmd_migration_entry()
3215 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
3219 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
3224 set_pmd_at(mm, address, pvmw->pmd, pmdval); in set_pmd_migration_entry()
3225 return -EBUSY; in set_pmd_migration_entry()
3245 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
3255 struct vm_area_struct *vma = pvmw->vma; in remove_migration_pmd()
3256 struct mm_struct *mm = vma->vm_mm; in remove_migration_pmd()
3257 unsigned long address = pvmw->address; in remove_migration_pmd()
3262 if (!(pvmw->pmd && !pvmw->pte)) in remove_migration_pmd()
3265 entry = pmd_to_swp_entry(*pvmw->pmd); in remove_migration_pmd()
3267 pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot)); in remove_migration_pmd()
3268 if (pmd_swp_soft_dirty(*pvmw->pmd)) in remove_migration_pmd()
3272 if (pmd_swp_uffd_wp(*pvmw->pmd)) in remove_migration_pmd()
3276 /* NOTE: this may contain setting soft-dirty on some archs */ in remove_migration_pmd()
3291 set_pmd_at(mm, haddr, pvmw->pmd, pmde); in remove_migration_pmd()
3293 /* No need to invalidate - it was non-present before */ in remove_migration_pmd()
3294 update_mmu_cache_pmd(vma, address, pvmw->pmd); in remove_migration_pmd()