| /mm/ |
| D | pgtable-generic.c | 106 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
109 int changed = !pmd_same(*pmdp, entry); in pmdp_set_access_flags()
112 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_set_access_flags()
121 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
125 young = pmdp_test_and_clear_young(vma, address, pmdp); in pmdp_clear_flush_young()
134 pmd_t *pmdp) in pmdp_huge_clear_flush() argument
138 VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_clear_flush()
139 !pmd_devmap(*pmdp)); in pmdp_huge_clear_flush()
140 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); in pmdp_huge_clear_flush()
161 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
[all …]
|
| D | debug_vm_pgtable.c | 70 pmd_t *pmdp; member
253 pgtable_trans_huge_deposit(args->mm, args->pmdp, args->start_ptep); in pmd_advanced_tests()
256 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); in pmd_advanced_tests()
258 pmdp_set_wrprotect(args->mm, vaddr, args->pmdp); in pmd_advanced_tests()
259 pmd = READ_ONCE(*args->pmdp); in pmd_advanced_tests()
261 pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp); in pmd_advanced_tests()
262 pmd = READ_ONCE(*args->pmdp); in pmd_advanced_tests()
268 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); in pmd_advanced_tests()
272 pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1); in pmd_advanced_tests()
273 pmd = READ_ONCE(*args->pmdp); in pmd_advanced_tests()
[all …]
|
| D | hmm.c | 224 unsigned long end, pmd_t *pmdp, pte_t *ptep, in hmm_vma_handle_pte() argument
279 migration_entry_wait(walk->mm, pmdp, addr); in hmm_vma_handle_pte()
320 static int hmm_vma_walk_pmd(pmd_t *pmdp, in hmm_vma_walk_pmd() argument
335 pmd = READ_ONCE(*pmdp); in hmm_vma_walk_pmd()
342 pmd_migration_entry_wait(walk->mm, pmdp); in hmm_vma_walk_pmd()
364 pmd = pmd_read_atomic(pmdp); in hmm_vma_walk_pmd()
384 ptep = pte_offset_map(pmdp, addr); in hmm_vma_walk_pmd()
388 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, hmm_pfns); in hmm_vma_walk_pmd()
|
| D | migrate.c | 2264 static int migrate_vma_collect_pmd(pmd_t *pmdp, in migrate_vma_collect_pmd() argument
2277 if (pmd_none(*pmdp)) in migrate_vma_collect_pmd()
2280 if (pmd_trans_huge(*pmdp)) { in migrate_vma_collect_pmd()
2283 ptl = pmd_lock(mm, pmdp); in migrate_vma_collect_pmd()
2284 if (unlikely(!pmd_trans_huge(*pmdp))) { in migrate_vma_collect_pmd()
2289 page = pmd_page(*pmdp); in migrate_vma_collect_pmd()
2292 split_huge_pmd(vma, pmdp, addr); in migrate_vma_collect_pmd()
2293 if (pmd_trans_unstable(pmdp)) in migrate_vma_collect_pmd()
2310 if (pmd_none(*pmdp)) in migrate_vma_collect_pmd()
2316 if (unlikely(pmd_bad(*pmdp))) in migrate_vma_collect_pmd()
[all …]
|
| D | gup.c | 2329 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, in gup_pte_range() argument
2376 if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) || in gup_pte_range()
2422 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, in gup_pte_range() argument
2463 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, in __gup_device_huge_pmd() argument
2474 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { in __gup_device_huge_pmd()
2499 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, in __gup_device_huge_pmd() argument
2600 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, in gup_huge_pmd() argument
2613 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags, in gup_huge_pmd()
2624 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { in gup_huge_pmd()
2701 pmd_t *pmdp; in gup_pmd_range() local
[all …]
|
| D | memory-failure.c | 604 static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, in check_hwpoisoned_pmd_entry() argument
607 pmd_t pmd = *pmdp; in check_hwpoisoned_pmd_entry()
622 static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, in check_hwpoisoned_pmd_entry() argument
629 static int hwpoison_pte_range(pmd_t *pmdp, unsigned long addr, in hwpoison_pte_range() argument
637 ptl = pmd_trans_huge_lock(pmdp, walk->vma); in hwpoison_pte_range()
639 ret = check_hwpoisoned_pmd_entry(pmdp, addr, hwp); in hwpoison_pte_range()
644 if (pmd_trans_unstable(pmdp)) in hwpoison_pte_range()
647 mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp, in hwpoison_pte_range()
|