Lines Matching +full:tlb +full:- +full:split
1 // SPDX-License-Identifier: GPL-2.0-only
37 #include <asm/tlb.h>
68 return transhuge_vma_enabled(vma, vma->vm_flags) && vma->vm_file && in file_thp_enabled()
69 !inode_is_open_for_write(vma->vm_file->f_inode) && in file_thp_enabled()
70 (vma->vm_flags & VM_EXEC); in file_thp_enabled()
76 unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE; in transparent_hugepage_active()
129 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
135 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
143 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_put_huge_zero_page()
202 ret = -EINVAL; in enabled_store()
234 return -EINVAL; in single_hugepage_flag_store()
288 return -EINVAL; in defrag_store()
340 return -ENOMEM; in hugepage_init_sysfs()
392 return -EINVAL; in hugepage_init()
400 * we use page->mapping and page->index in second tail page in hugepage_init()
422 * where the extra memory used could hurt more than TLB overhead in hugepage_init()
425 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { in hugepage_init()
481 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pmd_mkwrite()
489 struct mem_cgroup *memcg = compound_head(page)->mem_cgroup; in get_deferred_split_queue()
493 return &memcg->deferred_split_queue; in get_deferred_split_queue()
495 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
502 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
509 * we use page->mapping and page->indexlru in second tail page in prep_transhuge_page()
536 if (off_end <= off_align || (off_end - off_align) < size) in __thp_get_unmapped_area()
543 ret = current->mm->get_unmapped_area(filp, addr, len_pad, in __thp_get_unmapped_area()
560 ret += (off - ret) & (size - 1); in __thp_get_unmapped_area()
570 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) in thp_get_unmapped_area()
577 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); in thp_get_unmapped_area()
584 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
586 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
591 if (mem_cgroup_charge(page, vma->vm_mm, gfp)) { in __do_huge_pmd_anonymous_page()
599 pgtable = pte_alloc_one(vma->vm_mm); in __do_huge_pmd_anonymous_page()
605 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
613 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
614 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
619 ret = check_stable_address_space(vma->vm_mm); in __do_huge_pmd_anonymous_page()
627 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
629 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
635 entry = mk_huge_pmd(page, vma->vm_page_prot); in __do_huge_pmd_anonymous_page()
639 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
640 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in __do_huge_pmd_anonymous_page()
641 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
642 mm_inc_nr_ptes(vma->vm_mm); in __do_huge_pmd_anonymous_page()
643 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
645 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); in __do_huge_pmd_anonymous_page()
650 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
653 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
670 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE); in alloc_hugepage_direct_gfpmask()
702 entry = mk_pmd(zero_page, vma->vm_page_prot); in set_huge_zero_page()
713 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_anonymous_page()
716 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_anonymous_page()
722 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) in do_huge_pmd_anonymous_page()
724 if (!(vmf->flags & FAULT_FLAG_WRITE) && in do_huge_pmd_anonymous_page()
725 !mm_forbids_zeropage(vma->vm_mm) && in do_huge_pmd_anonymous_page()
730 pgtable = pte_alloc_one(vma->vm_mm); in do_huge_pmd_anonymous_page()
733 zero_page = mm_get_huge_zero_page(vma->vm_mm); in do_huge_pmd_anonymous_page()
735 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
739 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
741 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
742 ret = check_stable_address_space(vma->vm_mm); in do_huge_pmd_anonymous_page()
744 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
745 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
747 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
748 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
752 set_huge_zero_page(pgtable, vma->vm_mm, vma, in do_huge_pmd_anonymous_page()
753 haddr, vmf->pmd, zero_page); in do_huge_pmd_anonymous_page()
754 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
757 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
758 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
776 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pmd()
820 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
828 * @pgprot != @vmf->vma->vm_page_prot.
835 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_pfn_pmd_prot()
836 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pmd_prot()
844 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pmd_prot()
846 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pmd_prot()
848 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pmd_prot()
850 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pmd_prot()
854 pgtable = pte_alloc_one(vma->vm_mm); in vmf_insert_pfn_pmd_prot()
861 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); in vmf_insert_pfn_pmd_prot()
869 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pud_mkwrite()
877 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pud()
911 * vmf_insert_pfn_pud_prot - insert a pud size pfn
919 * @pgprot != @vmf->vma->vm_page_prot.
926 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_pfn_pud_prot()
927 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pud_prot()
934 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pud_prot()
936 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pud_prot()
938 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pud_prot()
940 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pud_prot()
945 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); in vmf_insert_pfn_pud_prot()
968 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pmd()
974 * When we COW a devmap PMD entry, we split it into PTEs, so we should in follow_devmap_pmd()
1000 return ERR_PTR(-EEXIST); in follow_devmap_pmd()
1005 return ERR_PTR(-EFAULT); in follow_devmap_pmd()
1008 page = ERR_PTR(-ENOMEM); in follow_devmap_pmd()
1021 int ret = -ENOMEM; in copy_huge_pmd()
1023 /* Skip if can be re-fill on fault */ in copy_huge_pmd()
1035 ret = -EAGAIN; in copy_huge_pmd()
1069 * under splitting since we don't split the page itself, only pmd to in copy_huge_pmd()
1086 * If this page is a potentially pinned page, split and retry the fault in copy_huge_pmd()
1090 * random page during the coming copy-on-write. in copy_huge_pmd()
1092 if (unlikely(is_cow_mapping(src_vma->vm_flags) && in copy_huge_pmd()
1093 atomic_read(&src_mm->has_pinned) && in copy_huge_pmd()
1099 return -EAGAIN; in copy_huge_pmd()
1140 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pud()
1168 return ERR_PTR(-EEXIST); in follow_devmap_pud()
1173 return ERR_PTR(-EFAULT); in follow_devmap_pud()
1176 page = ERR_PTR(-ENOMEM); in follow_devmap_pud()
1193 ret = -EAGAIN; in copy_huge_pud()
1200 * under splitting since we don't split the page itself, only pud to in copy_huge_pud()
1208 if (unlikely(is_cow_mapping(vma->vm_flags) && in copy_huge_pud()
1209 atomic_read(&src_mm->has_pinned) && in copy_huge_pud()
1214 return -EAGAIN; in copy_huge_pud()
1232 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pud_set_accessed()
1234 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); in huge_pud_set_accessed()
1235 if (unlikely(!pud_same(*vmf->pud, orig_pud))) in huge_pud_set_accessed()
1241 haddr = vmf->address & HPAGE_PUD_MASK; in huge_pud_set_accessed()
1242 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) in huge_pud_set_accessed()
1243 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); in huge_pud_set_accessed()
1246 spin_unlock(vmf->ptl); in huge_pud_set_accessed()
1254 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pmd_set_accessed()
1256 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1257 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) in huge_pmd_set_accessed()
1263 haddr = vmf->address & HPAGE_PMD_MASK; in huge_pmd_set_accessed()
1264 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) in huge_pmd_set_accessed()
1265 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); in huge_pmd_set_accessed()
1268 spin_unlock(vmf->ptl); in huge_pmd_set_accessed()
1273 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_wp_page()
1275 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_wp_page()
1277 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1278 VM_BUG_ON_VMA(!vma->anon_vma, vma); in do_huge_pmd_wp_page()
1283 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1285 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1286 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1296 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1298 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1299 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1300 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1316 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1317 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1319 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1324 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1326 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in do_huge_pmd_wp_page()
1345 struct mm_struct *mm = vma->vm_mm; in follow_trans_huge_pmd()
1355 return ERR_PTR(-EFAULT); in follow_trans_huge_pmd()
1365 return ERR_PTR(-ENOMEM); in follow_trans_huge_pmd()
1370 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { in follow_trans_huge_pmd()
1372 * We don't mlock() pte-mapped THPs. This way we can avoid in follow_trans_huge_pmd()
1373 * leaking mlocked pages into non-VM_LOCKED VMAs. in follow_trans_huge_pmd()
1378 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for in follow_trans_huge_pmd()
1382 * mlocking read-only mapping shared over fork(). We skip in follow_trans_huge_pmd()
1394 if (PageDoubleMap(page) || !page->mapping) in follow_trans_huge_pmd()
1398 if (page->mapping && !PageDoubleMap(page)) in follow_trans_huge_pmd()
1413 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_numa_page()
1416 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_numa_page()
1418 int target_nid, last_cpupid = -1; in do_huge_pmd_numa_page()
1424 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1425 if (unlikely(!pmd_same(pmd, *vmf->pmd))) in do_huge_pmd_numa_page()
1433 if (unlikely(pmd_trans_migrating(*vmf->pmd))) { in do_huge_pmd_numa_page()
1434 page = pmd_page(*vmf->pmd); in do_huge_pmd_numa_page()
1437 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1473 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1483 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1487 spin_lock(vmf->ptl); in do_huge_pmd_numa_page()
1488 if (unlikely(!pmd_same(pmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1510 * We are not sure a pending tlb flush here is for a huge page in do_huge_pmd_numa_page()
1511 * mapping or not. Hence use the tlb range variant in do_huge_pmd_numa_page()
1513 if (mm_tlb_flush_pending(vma->vm_mm)) { in do_huge_pmd_numa_page()
1518 * MMU pagetables (with ->invalidate_range()). The in do_huge_pmd_numa_page()
1520 * internally calls ->invalidate_range()) in in do_huge_pmd_numa_page()
1524 mmu_notifier_invalidate_range(vma->vm_mm, haddr, in do_huge_pmd_numa_page()
1532 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1534 migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma, in do_huge_pmd_numa_page()
1535 vmf->pmd, pmd, vmf->address, page, target_nid); in do_huge_pmd_numa_page()
1546 pmd = pmd_modify(pmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1550 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
1551 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
1554 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1571 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in madvise_free_huge_pmd() argument
1577 struct mm_struct *mm = tlb->mm; in madvise_free_huge_pmd()
1580 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_free_huge_pmd()
1608 * If user want to discard part-pages of THP, split it so MADV_FREE in madvise_free_huge_pmd()
1611 if (next - addr != HPAGE_PMD_SIZE) { in madvise_free_huge_pmd()
1630 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_free_huge_pmd()
1650 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pmd() argument
1656 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in zap_huge_pmd()
1668 tlb->fullmm); in zap_huge_pmd()
1669 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1672 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1675 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); in zap_huge_pmd()
1677 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1679 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); in zap_huge_pmd()
1700 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1701 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); in zap_huge_pmd()
1704 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1705 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); in zap_huge_pmd()
1710 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); in zap_huge_pmd()
1721 * With split pmd lock we also need to move preallocated in pmd_move_must_withdraw()
1746 struct mm_struct *mm = vma->vm_mm; in move_huge_pmd()
1791 * - 0 if PMD could not be locked
1792 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1793 * - HPAGE_PMD_NR is protections changed and TLB flush necessary
1798 struct mm_struct *mm = vma->vm_mm; in change_huge_pmd()
1838 * Avoid trapping faults against the zero page. The read-only in change_huge_pmd()
1839 * data is likely to be read-cached on the local CPU and in change_huge_pmd()
1861 * // pmd is re-established in change_huge_pmd()
1902 ptl = pmd_lock(vma->vm_mm, pmd); in __pmd_trans_huge_lock()
1920 ptl = pud_lock(vma->vm_mm, pud); in __pud_trans_huge_lock()
1928 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pud() argument
1942 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm); in zap_huge_pud()
1943 tlb_remove_pud_tlb_entry(tlb, pud, addr); in zap_huge_pud()
1958 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pud_locked()
1959 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); in __split_huge_pud_locked()
1973 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in __split_huge_pud()
1977 ptl = pud_lock(vma->vm_mm, pud); in __split_huge_pud()
1985 * No need to double call mmu_notifier->invalidate_range() callback as in __split_huge_pud()
1995 struct mm_struct *mm = vma->vm_mm; in __split_huge_zero_page_pmd()
2015 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); in __split_huge_zero_page_pmd()
2029 struct mm_struct *mm = vma->vm_mm; in __split_huge_pmd_locked()
2038 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pmd_locked()
2039 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); in __split_huge_pmd_locked()
2069 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); in __split_huge_pmd_locked()
2088 * whole access to the hugepage during the split (which happens in in __split_huge_pmd_locked()
2089 * place). If we overwrite the pmd with the not-huge version pointing in __split_huge_pmd_locked()
2091 * free), userland could trigger a small page size TLB miss on the in __split_huge_pmd_locked()
2092 * small sized TLB while the hugepage TLB entry is still established in in __split_huge_pmd_locked()
2093 * the huge TLB. Some CPU doesn't like that. in __split_huge_pmd_locked()
2097 * loaded in the two TLB is identical (which should be the case here). in __split_huge_pmd_locked()
2098 * But it is generally safer to never allow small and huge TLB entries in __split_huge_pmd_locked()
2102 * must remain set at all times on the pmd until the split is complete in __split_huge_pmd_locked()
2103 * for this pmd), then we flush the SMP TLB and finally we write the in __split_huge_pmd_locked()
2104 * non-huge version of the pmd entry with pmd_populate. in __split_huge_pmd_locked()
2128 page_ref_add(page, HPAGE_PMD_NR - 1); in __split_huge_pmd_locked()
2153 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); in __split_huge_pmd_locked()
2175 * false-negative page_mapped(). in __split_huge_pmd_locked()
2184 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { in __split_huge_pmd_locked()
2215 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in __split_huge_pmd()
2219 ptl = pmd_lock(vma->vm_mm, pmd); in __split_huge_pmd()
2271 * No need to double call mmu_notifier->invalidate_range() callback. in __split_huge_pmd()
2278 * 3) Split a huge pmd into pte pointing to the same page. No need in __split_huge_pmd()
2279 * to invalidate secondary tlb entry they are all still valid. in __split_huge_pmd()
2281 * to call mmu_notifier->invalidate_range() in __split_huge_pmd()
2294 pgd = pgd_offset(vma->vm_mm, address); in split_huge_pmd_address()
2318 * previously contain an hugepage: check if we need to split in vma_adjust_trans_huge()
2322 (start & HPAGE_PMD_MASK) >= vma->vm_start && in vma_adjust_trans_huge()
2323 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) in vma_adjust_trans_huge()
2328 * previously contain an hugepage: check if we need to split in vma_adjust_trans_huge()
2332 (end & HPAGE_PMD_MASK) >= vma->vm_start && in vma_adjust_trans_huge()
2333 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) in vma_adjust_trans_huge()
2337 * If we're also updating the vma->vm_next->vm_start, if the new in vma_adjust_trans_huge()
2338 * vm_next->vm_start isn't hpage aligned and it could previously in vma_adjust_trans_huge()
2339 * contain an hugepage: check if we need to split an huge pmd. in vma_adjust_trans_huge()
2342 struct vm_area_struct *next = vma->vm_next; in vma_adjust_trans_huge()
2343 unsigned long nstart = next->vm_start; in vma_adjust_trans_huge()
2346 (nstart & HPAGE_PMD_MASK) >= next->vm_start && in vma_adjust_trans_huge()
2347 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) in vma_adjust_trans_huge()
2383 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); in __split_huge_page_tail()
2391 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; in __split_huge_page_tail()
2392 page_tail->flags |= (head->flags & in __split_huge_page_tail()
2407 /* ->mapping in first tail page is compound_mapcount */ in __split_huge_page_tail()
2408 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, in __split_huge_page_tail()
2410 page_tail->mapping = head->mapping; in __split_huge_page_tail()
2411 page_tail->index = head->index + tail; in __split_huge_page_tail()
2413 /* Page flags must be visible before we make the page non-compound. */ in __split_huge_page_tail()
2437 * pages to show after the currently processed elements - e.g. in __split_huge_page_tail()
2464 xa_lock(&swap_cache->i_pages); in __split_huge_page()
2467 for (i = nr - 1; i >= 1; i--) { in __split_huge_page()
2474 shmem_uncharge(head->mapping->host, 1); in __split_huge_page()
2477 __xa_store(&head->mapping->i_pages, head[i].index, in __split_huge_page()
2480 __xa_store(&swap_cache->i_pages, offset + i, in __split_huge_page()
2494 xa_unlock(&swap_cache->i_pages); in __split_huge_page()
2501 xa_unlock(&head->mapping->i_pages); in __split_huge_page()
2504 spin_unlock_irqrestore(&pgdat->lru_lock, flags); in __split_huge_page()
2525 * of the tail pages after the split is complete. in __split_huge_page()
2538 return atomic_read(&page->_mapcount) + 1; in total_mapcount()
2549 return ret - compound * nr; in total_mapcount()
2551 ret -= nr; in total_mapcount()
2558 * accuracy is primarily needed to know if copy-on-write faults can
2559 * reuse the page and change the mapping to read-write instead of
2570 * anon_vma of the transparent hugepage can become the vma->anon_vma
2575 * page_trans_huge_mapcount() in the copy-on-write faults where we
2587 mapcount = atomic_read(&page->_mapcount) + 1; in page_trans_huge_mapcount()
2602 ret -= 1; in page_trans_huge_mapcount()
2603 _total_mapcount -= thp_nr_pages(page); in page_trans_huge_mapcount()
2613 /* Racy check whether the huge page can be split */
2625 return total_mapcount(page) == page_count(page) - extra_pins - 1; in can_split_huge_page()
2630 * subpage of huge page to split. Split doesn't change the position of @page.
2632 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2643 * Returns 0 if the hugepage is split successfully.
2644 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2663 return -EBUSY; in split_huge_page_to_list()
2671 * is taken to serialise against parallel split or collapse in split_huge_page_to_list()
2676 ret = -EBUSY; in split_huge_page_to_list()
2679 end = -1; in split_huge_page_to_list()
2683 mapping = head->mapping; in split_huge_page_to_list()
2687 ret = -EBUSY; in split_huge_page_to_list()
2696 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, in split_huge_page_to_list()
2701 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in split_huge_page_to_list()
2705 * Racy check if we can split the page, before unmap_page() will in split_huge_page_to_list()
2706 * split PMDs in split_huge_page_to_list()
2709 ret = -EBUSY; in split_huge_page_to_list()
2716 spin_lock_irqsave(&pgdata->lru_lock, flags); in split_huge_page_to_list()
2719 XA_STATE(xas, &mapping->i_pages, page_index(head)); in split_huge_page_to_list()
2725 xa_lock(&mapping->i_pages); in split_huge_page_to_list()
2730 /* Prevent deferred_split_scan() touching ->_refcount */ in split_huge_page_to_list()
2731 spin_lock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2734 ds_queue->split_queue_len--; in split_huge_page_to_list()
2737 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2748 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2751 xa_unlock(&mapping->i_pages); in split_huge_page_to_list()
2752 spin_unlock_irqrestore(&pgdata->lru_lock, flags); in split_huge_page_to_list()
2754 ret = -EBUSY; in split_huge_page_to_list()
2774 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in free_transhuge_page()
2776 ds_queue->split_queue_len--; in free_transhuge_page()
2779 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in free_transhuge_page()
2787 struct mem_cgroup *memcg = compound_head(page)->mem_cgroup; in deferred_split_huge_page()
2795 * this may cause a race condition to corrupt deferred split queue. in deferred_split_huge_page()
2806 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_huge_page()
2809 list_add_tail(page_deferred_list(page), &ds_queue->split_queue); in deferred_split_huge_page()
2810 ds_queue->split_queue_len++; in deferred_split_huge_page()
2817 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_huge_page()
2823 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_count()
2824 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_count()
2827 if (sc->memcg) in deferred_split_count()
2828 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_count()
2830 return READ_ONCE(ds_queue->split_queue_len); in deferred_split_count()
2836 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_scan()
2837 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_scan()
2841 int split = 0; in deferred_split_scan() local
2844 if (sc->memcg) in deferred_split_scan()
2845 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_scan()
2848 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2850 list_for_each_safe(pos, next, &ds_queue->split_queue) { in deferred_split_scan()
2858 ds_queue->split_queue_len--; in deferred_split_scan()
2860 if (!--sc->nr_to_scan) in deferred_split_scan()
2863 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2871 split++; in deferred_split_scan()
2877 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2878 list_splice_tail(&list, &ds_queue->split_queue); in deferred_split_scan()
2879 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2882 * Stop shrinker if we didn't split any page, but the queue is empty. in deferred_split_scan()
2885 if (!split && list_empty(&ds_queue->split_queue)) in deferred_split_scan()
2887 return split; in deferred_split_scan()
2904 unsigned long total = 0, split = 0; in split_huge_pages_set() local
2907 return -EINVAL; in split_huge_pages_set()
2911 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { in split_huge_pages_set()
2928 split++; in split_huge_pages_set()
2935 pr_info("%lu of %lu THP split\n", split, total); in split_huge_pages_set()
2955 struct vm_area_struct *vma = pvmw->vma; in set_pmd_migration_entry()
2956 struct mm_struct *mm = vma->vm_mm; in set_pmd_migration_entry()
2957 unsigned long address = pvmw->address; in set_pmd_migration_entry()
2962 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
2966 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
2973 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
2980 struct vm_area_struct *vma = pvmw->vma; in remove_migration_pmd()
2981 struct mm_struct *mm = vma->vm_mm; in remove_migration_pmd()
2982 unsigned long address = pvmw->address; in remove_migration_pmd()
2987 if (!(pvmw->pmd && !pvmw->pte)) in remove_migration_pmd()
2990 entry = pmd_to_swp_entry(*pvmw->pmd); in remove_migration_pmd()
2992 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); in remove_migration_pmd()
2993 if (pmd_swp_soft_dirty(*pvmw->pmd)) in remove_migration_pmd()
2997 if (pmd_swp_uffd_wp(*pvmw->pmd)) in remove_migration_pmd()
3005 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); in remove_migration_pmd()
3006 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) in remove_migration_pmd()
3008 update_mmu_cache_pmd(vma, address, pvmw->pmd); in remove_migration_pmd()