Lines Matching +full:tlb +full:- +full:split
1 // SPDX-License-Identifier: GPL-2.0-only
37 #include <asm/tlb.h>
68 return transhuge_vma_enabled(vma, vma->vm_flags) && vma->vm_file && in file_thp_enabled()
69 !inode_is_open_for_write(vma->vm_file->f_inode) && in file_thp_enabled()
70 (vma->vm_flags & VM_EXEC); in file_thp_enabled()
76 unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE; in transparent_hugepage_active()
129 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
135 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
143 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_put_huge_zero_page()
202 ret = -EINVAL; in enabled_store()
234 return -EINVAL; in single_hugepage_flag_store()
288 return -EINVAL; in defrag_store()
340 return -ENOMEM; in hugepage_init_sysfs()
392 return -EINVAL; in hugepage_init()
400 * we use page->mapping and page->index in second tail page in hugepage_init()
422 * where the extra memory used could hurt more than TLB overhead in hugepage_init()
425 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { in hugepage_init()
481 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pmd_mkwrite()
489 struct mem_cgroup *memcg = compound_head(page)->mem_cgroup; in get_deferred_split_queue()
493 return &memcg->deferred_split_queue; in get_deferred_split_queue()
495 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
502 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
509 * we use page->mapping and page->indexlru in second tail page in prep_transhuge_page()
536 if (off_end <= off_align || (off_end - off_align) < size) in __thp_get_unmapped_area()
543 ret = current->mm->get_unmapped_area(filp, addr, len_pad, in __thp_get_unmapped_area()
560 ret += (off - ret) & (size - 1); in __thp_get_unmapped_area()
570 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) in thp_get_unmapped_area()
577 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); in thp_get_unmapped_area()
584 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
586 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
591 if (mem_cgroup_charge(page, vma->vm_mm, gfp)) { in __do_huge_pmd_anonymous_page()
599 pgtable = pte_alloc_one(vma->vm_mm); in __do_huge_pmd_anonymous_page()
605 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
613 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
614 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
619 ret = check_stable_address_space(vma->vm_mm); in __do_huge_pmd_anonymous_page()
627 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
629 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
635 entry = mk_huge_pmd(page, vma->vm_page_prot); in __do_huge_pmd_anonymous_page()
639 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
640 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in __do_huge_pmd_anonymous_page()
641 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
642 mm_inc_nr_ptes(vma->vm_mm); in __do_huge_pmd_anonymous_page()
643 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
645 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); in __do_huge_pmd_anonymous_page()
650 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
653 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
670 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE); in alloc_hugepage_direct_gfpmask()
702 entry = mk_pmd(zero_page, vma->vm_page_prot); in set_huge_zero_page()
713 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_anonymous_page()
716 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_anonymous_page()
722 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) in do_huge_pmd_anonymous_page()
724 if (!(vmf->flags & FAULT_FLAG_WRITE) && in do_huge_pmd_anonymous_page()
725 !mm_forbids_zeropage(vma->vm_mm) && in do_huge_pmd_anonymous_page()
730 pgtable = pte_alloc_one(vma->vm_mm); in do_huge_pmd_anonymous_page()
733 zero_page = mm_get_huge_zero_page(vma->vm_mm); in do_huge_pmd_anonymous_page()
735 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
739 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
741 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
742 ret = check_stable_address_space(vma->vm_mm); in do_huge_pmd_anonymous_page()
744 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
745 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
747 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
748 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
752 set_huge_zero_page(pgtable, vma->vm_mm, vma, in do_huge_pmd_anonymous_page()
753 haddr, vmf->pmd, zero_page); in do_huge_pmd_anonymous_page()
754 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
757 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
758 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
776 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pmd()
820 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
828 * @pgprot != @vmf->vma->vm_page_prot.
835 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_pfn_pmd_prot()
836 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pmd_prot()
844 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pmd_prot()
846 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pmd_prot()
848 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pmd_prot()
850 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pmd_prot()
854 pgtable = pte_alloc_one(vma->vm_mm); in vmf_insert_pfn_pmd_prot()
861 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); in vmf_insert_pfn_pmd_prot()
869 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pud_mkwrite()
877 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pud()
911 * vmf_insert_pfn_pud_prot - insert a pud size pfn
919 * @pgprot != @vmf->vma->vm_page_prot.
926 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_pfn_pud_prot()
927 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pud_prot()
934 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pud_prot()
936 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pud_prot()
938 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pud_prot()
940 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pud_prot()
945 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); in vmf_insert_pfn_pud_prot()
968 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pmd()
974 * When we COW a devmap PMD entry, we split it into PTEs, so we should in follow_devmap_pmd()
1000 return ERR_PTR(-EEXIST); in follow_devmap_pmd()
1005 return ERR_PTR(-EFAULT); in follow_devmap_pmd()
1008 page = ERR_PTR(-ENOMEM); in follow_devmap_pmd()
1021 int ret = -ENOMEM; in copy_huge_pmd()
1023 /* Skip if can be re-fill on fault */ in copy_huge_pmd()
1035 ret = -EAGAIN; in copy_huge_pmd()
1069 * under splitting since we don't split the page itself, only pmd to in copy_huge_pmd()
1086 * If this page is a potentially pinned page, split and retry the fault in copy_huge_pmd()
1090 * random page during the coming copy-on-write. in copy_huge_pmd()
1092 if (unlikely(is_cow_mapping(src_vma->vm_flags) && in copy_huge_pmd()
1093 atomic_read(&src_mm->has_pinned) && in copy_huge_pmd()
1099 return -EAGAIN; in copy_huge_pmd()
1140 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pud()
1168 return ERR_PTR(-EEXIST); in follow_devmap_pud()
1173 return ERR_PTR(-EFAULT); in follow_devmap_pud()
1176 page = ERR_PTR(-ENOMEM); in follow_devmap_pud()
1193 ret = -EAGAIN; in copy_huge_pud()
1200 * under splitting since we don't split the page itself, only pud to in copy_huge_pud()
1208 if (unlikely(is_cow_mapping(vma->vm_flags) && in copy_huge_pud()
1209 atomic_read(&src_mm->has_pinned) && in copy_huge_pud()
1214 return -EAGAIN; in copy_huge_pud()
1232 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pud_set_accessed()
1234 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); in huge_pud_set_accessed()
1235 if (unlikely(!pud_same(*vmf->pud, orig_pud))) in huge_pud_set_accessed()
1241 haddr = vmf->address & HPAGE_PUD_MASK; in huge_pud_set_accessed()
1242 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) in huge_pud_set_accessed()
1243 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); in huge_pud_set_accessed()
1246 spin_unlock(vmf->ptl); in huge_pud_set_accessed()
1254 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pmd_set_accessed()
1256 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1257 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) in huge_pmd_set_accessed()
1263 haddr = vmf->address & HPAGE_PMD_MASK; in huge_pmd_set_accessed()
1264 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) in huge_pmd_set_accessed()
1265 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); in huge_pmd_set_accessed()
1268 spin_unlock(vmf->ptl); in huge_pmd_set_accessed()
1273 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_wp_page()
1275 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_wp_page()
1277 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1278 VM_BUG_ON_VMA(!vma->anon_vma, vma); in do_huge_pmd_wp_page()
1283 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1285 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1286 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1296 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1298 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1299 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1300 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1316 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1317 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1319 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1324 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1326 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in do_huge_pmd_wp_page()
1345 struct mm_struct *mm = vma->vm_mm; in follow_trans_huge_pmd()
1355 return ERR_PTR(-EFAULT); in follow_trans_huge_pmd()
1365 return ERR_PTR(-ENOMEM); in follow_trans_huge_pmd()
1370 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { in follow_trans_huge_pmd()
1372 * We don't mlock() pte-mapped THPs. This way we can avoid in follow_trans_huge_pmd()
1373 * leaking mlocked pages into non-VM_LOCKED VMAs. in follow_trans_huge_pmd()
1378 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for in follow_trans_huge_pmd()
1382 * mlocking read-only mapping shared over fork(). We skip in follow_trans_huge_pmd()
1394 if (PageDoubleMap(page) || !page->mapping) in follow_trans_huge_pmd()
1398 if (page->mapping && !PageDoubleMap(page)) in follow_trans_huge_pmd()
1413 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_numa_page()
1416 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_numa_page()
1418 int target_nid, last_cpupid = -1; in do_huge_pmd_numa_page()
1424 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1425 if (unlikely(!pmd_same(pmd, *vmf->pmd))) in do_huge_pmd_numa_page()
1433 if (unlikely(pmd_trans_migrating(*vmf->pmd))) { in do_huge_pmd_numa_page()
1434 page = pmd_page(*vmf->pmd); in do_huge_pmd_numa_page()
1437 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1473 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1483 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1487 spin_lock(vmf->ptl); in do_huge_pmd_numa_page()
1488 if (unlikely(!pmd_same(pmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1510 * We are not sure a pending tlb flush here is for a huge page in do_huge_pmd_numa_page()
1511 * mapping or not. Hence use the tlb range variant in do_huge_pmd_numa_page()
1513 if (mm_tlb_flush_pending(vma->vm_mm)) { in do_huge_pmd_numa_page()
1518 * MMU pagetables (with ->invalidate_range()). The in do_huge_pmd_numa_page()
1520 * internally calls ->invalidate_range()) in in do_huge_pmd_numa_page()
1524 mmu_notifier_invalidate_range(vma->vm_mm, haddr, in do_huge_pmd_numa_page()
1532 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1534 migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma, in do_huge_pmd_numa_page()
1535 vmf->pmd, pmd, vmf->address, page, target_nid); in do_huge_pmd_numa_page()
1546 pmd = pmd_modify(pmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1550 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
1551 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
1554 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1571 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in madvise_free_huge_pmd() argument
1577 struct mm_struct *mm = tlb->mm; in madvise_free_huge_pmd()
1580 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_free_huge_pmd()
1608 * If user want to discard part-pages of THP, split it so MADV_FREE in madvise_free_huge_pmd()
1611 if (next - addr != HPAGE_PMD_SIZE) { in madvise_free_huge_pmd()
1630 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_free_huge_pmd()
1650 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pmd() argument
1656 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in zap_huge_pmd()
1668 tlb->fullmm); in zap_huge_pmd()
1669 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1672 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1675 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); in zap_huge_pmd()
1677 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1679 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); in zap_huge_pmd()
1700 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1701 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); in zap_huge_pmd()
1704 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1705 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); in zap_huge_pmd()
1710 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); in zap_huge_pmd()
1721 * With split pmd lock we also need to move preallocated in pmd_move_must_withdraw()
1746 struct mm_struct *mm = vma->vm_mm; in move_huge_pmd()
1791 * - 0 if PMD could not be locked
1792 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1793 * - HPAGE_PMD_NR is protections changed and TLB flush necessary
1798 struct mm_struct *mm = vma->vm_mm; in change_huge_pmd()
1838 * Avoid trapping faults against the zero page. The read-only in change_huge_pmd()
1839 * data is likely to be read-cached on the local CPU and in change_huge_pmd()
1861 * // pmd is re-established in change_huge_pmd()
1902 ptl = pmd_lock(vma->vm_mm, pmd); in __pmd_trans_huge_lock()
1920 ptl = pud_lock(vma->vm_mm, pud); in __pud_trans_huge_lock()
1928 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pud() argument
1942 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm); in zap_huge_pud()
1943 tlb_remove_pud_tlb_entry(tlb, pud, addr); in zap_huge_pud()
1958 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pud_locked()
1959 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); in __split_huge_pud_locked()
1973 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in __split_huge_pud()
1977 ptl = pud_lock(vma->vm_mm, pud); in __split_huge_pud()
1985 * No need to double call mmu_notifier->invalidate_range() callback as in __split_huge_pud()
1995 struct mm_struct *mm = vma->vm_mm; in __split_huge_zero_page_pmd()
2015 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); in __split_huge_zero_page_pmd()
2031 struct mm_struct *mm = vma->vm_mm; in __split_huge_pmd_locked()
2040 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pmd_locked()
2041 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); in __split_huge_pmd_locked()
2071 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); in __split_huge_pmd_locked()
2090 * whole access to the hugepage during the split (which happens in in __split_huge_pmd_locked()
2091 * place). If we overwrite the pmd with the not-huge version pointing in __split_huge_pmd_locked()
2093 * free), userland could trigger a small page size TLB miss on the in __split_huge_pmd_locked()
2094 * small sized TLB while the hugepage TLB entry is still established in in __split_huge_pmd_locked()
2095 * the huge TLB. Some CPU doesn't like that. in __split_huge_pmd_locked()
2099 * loaded in the two TLB is identical (which should be the case here). in __split_huge_pmd_locked()
2100 * But it is generally safer to never allow small and huge TLB entries in __split_huge_pmd_locked()
2104 * must remain set at all times on the pmd until the split is complete in __split_huge_pmd_locked()
2105 * for this pmd), then we flush the SMP TLB and finally we write the in __split_huge_pmd_locked()
2106 * non-huge version of the pmd entry with pmd_populate. in __split_huge_pmd_locked()
2130 page_ref_add(page, HPAGE_PMD_NR - 1); in __split_huge_pmd_locked()
2155 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); in __split_huge_pmd_locked()
2177 * false-negative page_mapped(). in __split_huge_pmd_locked()
2186 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { in __split_huge_pmd_locked()
2217 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in __split_huge_pmd()
2221 ptl = pmd_lock(vma->vm_mm, pmd); in __split_huge_pmd()
2273 * No need to double call mmu_notifier->invalidate_range() callback. in __split_huge_pmd()
2280 * 3) Split a huge pmd into pte pointing to the same page. No need in __split_huge_pmd()
2281 * to invalidate secondary tlb entry they are all still valid. in __split_huge_pmd()
2283 * to call mmu_notifier->invalidate_range() in __split_huge_pmd()
2296 pgd = pgd_offset(vma->vm_mm, address); in split_huge_pmd_address()
2320 * previously contain an hugepage: check if we need to split in vma_adjust_trans_huge()
2324 (start & HPAGE_PMD_MASK) >= vma->vm_start && in vma_adjust_trans_huge()
2325 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) in vma_adjust_trans_huge()
2330 * previously contain an hugepage: check if we need to split in vma_adjust_trans_huge()
2334 (end & HPAGE_PMD_MASK) >= vma->vm_start && in vma_adjust_trans_huge()
2335 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) in vma_adjust_trans_huge()
2339 * If we're also updating the vma->vm_next->vm_start, if the new in vma_adjust_trans_huge()
2340 * vm_next->vm_start isn't hpage aligned and it could previously in vma_adjust_trans_huge()
2341 * contain an hugepage: check if we need to split an huge pmd. in vma_adjust_trans_huge()
2344 struct vm_area_struct *next = vma->vm_next; in vma_adjust_trans_huge()
2345 unsigned long nstart = next->vm_start; in vma_adjust_trans_huge()
2348 (nstart & HPAGE_PMD_MASK) >= next->vm_start && in vma_adjust_trans_huge()
2349 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) in vma_adjust_trans_huge()
2385 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); in __split_huge_page_tail()
2393 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; in __split_huge_page_tail()
2394 page_tail->flags |= (head->flags & in __split_huge_page_tail()
2409 /* ->mapping in first tail page is compound_mapcount */ in __split_huge_page_tail()
2410 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, in __split_huge_page_tail()
2412 page_tail->mapping = head->mapping; in __split_huge_page_tail()
2413 page_tail->index = head->index + tail; in __split_huge_page_tail()
2415 /* Page flags must be visible before we make the page non-compound. */ in __split_huge_page_tail()
2439 * pages to show after the currently processed elements - e.g. in __split_huge_page_tail()
2466 xa_lock(&swap_cache->i_pages); in __split_huge_page()
2469 for (i = nr - 1; i >= 1; i--) { in __split_huge_page()
2476 shmem_uncharge(head->mapping->host, 1); in __split_huge_page()
2479 __xa_store(&head->mapping->i_pages, head[i].index, in __split_huge_page()
2482 __xa_store(&swap_cache->i_pages, offset + i, in __split_huge_page()
2496 xa_unlock(&swap_cache->i_pages); in __split_huge_page()
2503 xa_unlock(&head->mapping->i_pages); in __split_huge_page()
2506 spin_unlock_irqrestore(&pgdat->lru_lock, flags); in __split_huge_page()
2527 * of the tail pages after the split is complete. in __split_huge_page()
2540 return atomic_read(&page->_mapcount) + 1; in total_mapcount()
2551 return ret - compound * nr; in total_mapcount()
2553 ret -= nr; in total_mapcount()
2560 * accuracy is primarily needed to know if copy-on-write faults can
2561 * reuse the page and change the mapping to read-write instead of
2572 * anon_vma of the transparent hugepage can become the vma->anon_vma
2577 * page_trans_huge_mapcount() in the copy-on-write faults where we
2589 mapcount = atomic_read(&page->_mapcount) + 1; in page_trans_huge_mapcount()
2604 ret -= 1; in page_trans_huge_mapcount()
2605 _total_mapcount -= thp_nr_pages(page); in page_trans_huge_mapcount()
2615 /* Racy check whether the huge page can be split */
2627 return total_mapcount(page) == page_count(page) - extra_pins - 1; in can_split_huge_page()
2632 * subpage of huge page to split. Split doesn't change the position of @page.
2634 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2645 * Returns 0 if the hugepage is split successfully.
2646 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2665 return -EBUSY; in split_huge_page_to_list()
2673 * is taken to serialise against parallel split or collapse in split_huge_page_to_list()
2678 ret = -EBUSY; in split_huge_page_to_list()
2681 end = -1; in split_huge_page_to_list()
2685 mapping = head->mapping; in split_huge_page_to_list()
2689 ret = -EBUSY; in split_huge_page_to_list()
2698 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, in split_huge_page_to_list()
2703 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in split_huge_page_to_list()
2707 * Racy check if we can split the page, before unmap_page() will in split_huge_page_to_list()
2708 * split PMDs in split_huge_page_to_list()
2711 ret = -EBUSY; in split_huge_page_to_list()
2718 spin_lock_irqsave(&pgdata->lru_lock, flags); in split_huge_page_to_list()
2721 XA_STATE(xas, &mapping->i_pages, page_index(head)); in split_huge_page_to_list()
2727 xa_lock(&mapping->i_pages); in split_huge_page_to_list()
2732 /* Prevent deferred_split_scan() touching ->_refcount */ in split_huge_page_to_list()
2733 spin_lock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2736 ds_queue->split_queue_len--; in split_huge_page_to_list()
2739 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2750 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2753 xa_unlock(&mapping->i_pages); in split_huge_page_to_list()
2754 spin_unlock_irqrestore(&pgdata->lru_lock, flags); in split_huge_page_to_list()
2756 ret = -EBUSY; in split_huge_page_to_list()
2782 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in __page_unqueue_deferred_split()
2784 ds_queue->split_queue_len--; in __page_unqueue_deferred_split()
2787 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in __page_unqueue_deferred_split()
2801 struct mem_cgroup *memcg = compound_head(page)->mem_cgroup; in deferred_split_huge_page()
2809 * this may cause a race condition to corrupt deferred split queue. in deferred_split_huge_page()
2823 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_huge_page()
2826 list_add_tail(page_deferred_list(page), &ds_queue->split_queue); in deferred_split_huge_page()
2827 ds_queue->split_queue_len++; in deferred_split_huge_page()
2834 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_huge_page()
2840 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_count()
2841 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_count()
2844 if (sc->memcg) in deferred_split_count()
2845 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_count()
2847 return READ_ONCE(ds_queue->split_queue_len); in deferred_split_count()
2853 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_scan()
2854 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_scan()
2858 int split = 0; in deferred_split_scan() local
2861 if (sc->memcg) in deferred_split_scan()
2862 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_scan()
2865 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2867 list_for_each_safe(pos, next, &ds_queue->split_queue) { in deferred_split_scan()
2875 ds_queue->split_queue_len--; in deferred_split_scan()
2877 if (!--sc->nr_to_scan) in deferred_split_scan()
2880 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2888 split++; in deferred_split_scan()
2894 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2895 list_splice_tail(&list, &ds_queue->split_queue); in deferred_split_scan()
2896 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2899 * Stop shrinker if we didn't split any page, but the queue is empty. in deferred_split_scan()
2902 if (!split && list_empty(&ds_queue->split_queue)) in deferred_split_scan()
2904 return split; in deferred_split_scan()
2921 unsigned long total = 0, split = 0; in split_huge_pages_set() local
2924 return -EINVAL; in split_huge_pages_set()
2928 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { in split_huge_pages_set()
2945 split++; in split_huge_pages_set()
2952 pr_info("%lu of %lu THP split\n", split, total); in split_huge_pages_set()
2972 struct vm_area_struct *vma = pvmw->vma; in set_pmd_migration_entry()
2973 struct mm_struct *mm = vma->vm_mm; in set_pmd_migration_entry()
2974 unsigned long address = pvmw->address; in set_pmd_migration_entry()
2979 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
2983 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
2990 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
2997 struct vm_area_struct *vma = pvmw->vma; in remove_migration_pmd()
2998 struct mm_struct *mm = vma->vm_mm; in remove_migration_pmd()
2999 unsigned long address = pvmw->address; in remove_migration_pmd()
3004 if (!(pvmw->pmd && !pvmw->pte)) in remove_migration_pmd()
3007 entry = pmd_to_swp_entry(*pvmw->pmd); in remove_migration_pmd()
3009 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); in remove_migration_pmd()
3010 if (pmd_swp_soft_dirty(*pvmw->pmd)) in remove_migration_pmd()
3014 if (pmd_swp_uffd_wp(*pvmw->pmd)) in remove_migration_pmd()
3022 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); in remove_migration_pmd()
3023 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) in remove_migration_pmd()
3025 update_mmu_cache_pmd(vma, address, pvmw->pmd); in remove_migration_pmd()