/arch/arm64/include/asm/ |
D | pgalloc.h | 23 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument 25 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate() 28 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument 30 __pud_populate(pudp, __pa(pmdp), PMD_TYPE_TABLE); in pud_populate() 33 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument 41 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) in __p4d_populate() argument 43 set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); in __p4d_populate() 46 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) in p4d_populate() argument 48 __p4d_populate(p4dp, __pa(pudp), PUD_TYPE_TABLE); in p4d_populate() 51 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) in __p4d_populate() argument
|
D | tlb.h | 94 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, in __pud_free_tlb() argument 97 tlb_remove_table(tlb, virt_to_page(pudp)); in __pud_free_tlb()
|
/arch/powerpc/mm/nohash/ |
D | book3e_pgtable.c | 77 pud_t *pudp; in map_kernel_page() local 85 pudp = pud_alloc(&init_mm, p4dp, ea); in map_kernel_page() 86 if (!pudp) in map_kernel_page() 88 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page() 98 pudp = early_alloc_pgtable(PUD_TABLE_SIZE); in map_kernel_page() 99 p4d_populate(&init_mm, p4dp, pudp); in map_kernel_page() 101 pudp = pud_offset(p4dp, ea); in map_kernel_page() 102 if (pud_none(*pudp)) { in map_kernel_page() 104 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page() 106 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
|
/arch/s390/mm/ |
D | pageattr.c | 154 static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, in walk_pmd_level() argument 161 pmdp = pmd_offset(pudp, addr); in walk_pmd_level() 186 static int split_pud_page(pud_t *pudp, unsigned long addr) in split_pud_page() argument 196 pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; in split_pud_page() 197 ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); in split_pud_page() 198 nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); in split_pud_page() 209 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); in split_pud_page() 215 static void modify_pud_page(pud_t *pudp, unsigned long addr, in modify_pud_page() argument 218 pud_t new = *pudp; in modify_pud_page() 228 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); in modify_pud_page() [all …]
|
D | hugetlbpage.c | 183 pud_t *pudp = (pud_t *) ptep; in huge_ptep_get_and_clear() local 186 pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); in huge_ptep_get_and_clear() 197 pud_t *pudp; in huge_pte_alloc() local 203 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc() 204 if (pudp) { in huge_pte_alloc() 206 return (pte_t *) pudp; in huge_pte_alloc() 208 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc() 219 pud_t *pudp; in huge_pte_offset() local 226 pudp = pud_offset(p4dp, addr); in huge_pte_offset() 227 if (pud_present(*pudp)) { in huge_pte_offset() [all …]
|
D | pgtable.c | 478 unsigned long addr, pud_t *pudp) in pudp_idte_local() argument 481 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, in pudp_idte_local() 484 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); in pudp_idte_local() 488 unsigned long addr, pud_t *pudp) in pudp_idte_global() argument 491 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, in pudp_idte_global() 494 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); in pudp_idte_global() 500 __pmdp_csp((pmd_t *) pudp); in pudp_idte_global() 504 unsigned long addr, pud_t *pudp) in pudp_flush_direct() argument 508 old = *pudp; in pudp_flush_direct() 514 pudp_idte_local(mm, addr, pudp); in pudp_flush_direct() [all …]
|
/arch/arm64/mm/ |
D | mmu.c | 213 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, in init_pmd() argument 220 pmdp = pmd_set_fixmap_offset(pudp, addr); in init_pmd() 250 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, in alloc_init_cont_pmd() argument 256 pud_t pud = READ_ONCE(*pudp); in alloc_init_cont_pmd() 266 __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); in alloc_init_cont_pmd() 267 pud = READ_ONCE(*pudp); in alloc_init_cont_pmd() 281 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); in alloc_init_cont_pmd() 305 pud_t *pudp; in alloc_init_pud() local 324 pudp = pud_set_fixmap_offset(p4dp, addr); in alloc_init_pud() 326 pud_t old_pud = READ_ONCE(*pudp); in alloc_init_pud() [all …]
|
D | hugetlbpage.c | 108 pud_t *pudp; in find_num_contig() local 113 pudp = pud_offset(p4dp, addr); in find_num_contig() 114 pmdp = pmd_offset(pudp, addr); in find_num_contig() 260 pud_t *pudp; in huge_pte_alloc() local 266 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc() 267 if (!pudp) in huge_pte_alloc() 271 ptep = (pte_t *)pudp; in huge_pte_alloc() 273 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc() 287 if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp))) in huge_pte_alloc() 288 ptep = huge_pmd_share(mm, vma, addr, pudp); in huge_pte_alloc() [all …]
|
D | kasan_init.c | 75 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, in kasan_pmd_offset() argument 78 if (pud_none(READ_ONCE(*pudp))) { in kasan_pmd_offset() 82 __pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE); in kasan_pmd_offset() 85 return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); in kasan_pmd_offset() 118 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 122 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() 134 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate() local 138 kasan_pmd_populate(pudp, addr, next, node, early); in kasan_pud_populate() 139 } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); in kasan_pud_populate()
|
D | pageattr.c | 202 pud_t *pudp, pud; in kernel_page_present() local 218 pudp = pud_offset(p4dp, addr); in kernel_page_present() 219 pud = READ_ONCE(*pudp); in kernel_page_present() 225 pmdp = pmd_offset(pudp, addr); in kernel_page_present()
|
/arch/x86/include/asm/ |
D | pgtable-3level.h | 103 static inline void native_set_pud(pud_t *pudp, pud_t pud) in native_set_pud() argument 106 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); in native_set_pud() 108 set_64bit((unsigned long long *)(pudp), native_pud_val(pud)); in native_set_pud() 132 static inline void native_pud_clear(pud_t *pudp) in native_pud_clear() argument 136 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 138 set_pud(pudp, __pud(0)); in pud_clear() 232 static inline pud_t native_pudp_get_and_clear(pud_t *pudp) in native_pudp_get_and_clear() argument 234 union split_pud res, *orig = (union split_pud *)pudp; in native_pudp_get_and_clear() 237 pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0)); in native_pudp_get_and_clear()
|
D | pgtable-2level.h | 25 static inline void native_set_pud(pud_t *pudp, pud_t pud) in native_set_pud() argument 39 static inline void native_pud_clear(pud_t *pudp) in native_pud_clear() argument
|
/arch/riscv/include/asm/ |
D | pgtable-64.h | 53 static inline void set_pud(pud_t *pudp, pud_t pud) in set_pud() argument 55 *pudp = pud; in set_pud() 58 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 60 set_pud(pudp, __pud(0)); in pud_clear()
|
/arch/arm64/kernel/ |
D | hibernate.c | 187 pud_t *pudp; in trans_pgd_map_page() local 193 pudp = (void *)get_safe_page(GFP_ATOMIC); in trans_pgd_map_page() 194 if (!pudp) in trans_pgd_map_page() 196 pgd_populate(&init_mm, pgdp, pudp); in trans_pgd_map_page() 201 pudp = (void *)get_safe_page(GFP_ATOMIC); in trans_pgd_map_page() 202 if (!pudp) in trans_pgd_map_page() 204 p4d_populate(&init_mm, p4dp, pudp); in trans_pgd_map_page() 207 pudp = pud_offset(p4dp, dst_addr); in trans_pgd_map_page() 208 if (pud_none(READ_ONCE(*pudp))) { in trans_pgd_map_page() 212 pud_populate(&init_mm, pudp, pmdp); in trans_pgd_map_page() [all …]
|
/arch/powerpc/mm/book3s64/ |
D | radix_pgtable.c | 76 pud_t *pudp; in early_map_kernel_page() local 83 pudp = early_alloc_pgtable(PAGE_SIZE, nid, in early_map_kernel_page() 85 p4d_populate(&init_mm, p4dp, pudp); in early_map_kernel_page() 87 pudp = pud_offset(p4dp, ea); in early_map_kernel_page() 89 ptep = (pte_t *)pudp; in early_map_kernel_page() 92 if (pud_none(*pudp)) { in early_map_kernel_page() 95 pud_populate(&init_mm, pudp, pmdp); in early_map_kernel_page() 97 pmdp = pmd_offset(pudp, ea); in early_map_kernel_page() 128 pud_t *pudp; in __map_kernel_page() local 151 pudp = pud_alloc(&init_mm, p4dp, ea); in __map_kernel_page() [all …]
|
D | hash_pgtable.c | 150 pud_t *pudp; in hash__map_kernel_page() local 158 pudp = pud_alloc(&init_mm, p4dp, ea); in hash__map_kernel_page() 159 if (!pudp) in hash__map_kernel_page() 161 pmdp = pmd_alloc(&init_mm, pudp, ea); in hash__map_kernel_page()
|
/arch/arm/include/asm/ |
D | pgtable-3level.h | 121 #define pud_clear(pudp) \ argument 123 *pudp = __pud(0); \ 124 clean_pmd_entry(pudp); \ 127 #define set_pud(pudp, pud) \ argument 129 *pudp = pud; \ 130 flush_pmd_entry(pudp); \
|
D | pgtable-2level.h | 185 #define pud_clear(pudp) do { } while (0) argument 186 #define set_pud(pud,pudp) do { } while (0) argument
|
/arch/sparc/include/asm/ |
D | pgalloc_32.h | 29 static inline void pud_set(pud_t * pudp, pmd_t * pmdp) in pud_set() argument 33 set_pte((pte_t *)pudp, __pte((SRMMU_ET_PTD | (pa >> 4)))); in pud_set()
|
/arch/sparc/mm/ |
D | srmmu.c | 498 pud_t *pudp; in srmmu_mapioaddr() local 506 pudp = pud_offset(p4dp, virt_addr); in srmmu_mapioaddr() 507 pmdp = pmd_offset(pudp, virt_addr); in srmmu_mapioaddr() 537 pud_t *pudp; in srmmu_unmapioaddr() local 544 pudp = pud_offset(p4dp, virt_addr); in srmmu_unmapioaddr() 545 pmdp = pmd_offset(pudp, virt_addr); in srmmu_unmapioaddr() 684 pud_t *pudp; in srmmu_early_allocate_ptable_skeleton() local 691 pudp = pud_offset(p4dp, start); in srmmu_early_allocate_ptable_skeleton() 692 if (pud_none(*(pud_t *)__nocache_fix(pudp))) { in srmmu_early_allocate_ptable_skeleton() 698 pud_set(__nocache_fix(pudp), pmdp); in srmmu_early_allocate_ptable_skeleton() [all …]
|
D | fault_64.c | 83 pud_t *pudp; in get_user_insn() local 94 pudp = pud_offset(p4dp, tpc); in get_user_insn() 95 if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) in get_user_insn() 101 pmdp = pmd_offset(pudp, tpc); in get_user_insn()
|
/arch/x86/mm/ |
D | pgtable.c | 188 void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) in pud_populate() argument 194 set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT)); in pud_populate() 521 pud_t *pudp, pud_t entry, int dirty) in pudp_set_access_flags() argument 523 int changed = !pud_same(*pudp, entry); in pudp_set_access_flags() 528 set_pud(pudp, entry); in pudp_set_access_flags() 566 unsigned long addr, pud_t *pudp) in pudp_test_and_clear_young() argument 570 if (pud_young(*pudp)) in pudp_test_and_clear_young() 572 (unsigned long *)pudp); in pudp_test_and_clear_young()
|
/arch/m68k/include/asm/ |
D | motorola_pgtable.h | 125 static inline void pud_set(pud_t *pudp, pmd_t *pmdp) in pud_set() argument 127 pud_val(*pudp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pud_set() 159 #define pud_clear(pudp) ({ pud_val(*pudp) = 0; }) argument
|
/arch/powerpc/include/asm/nohash/64/ |
D | pgtable.h | 153 static inline void pud_set(pud_t *pudp, unsigned long val) in pud_set() argument 155 *pudp = __pud(val); in pud_set() 158 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 160 *pudp = __pud(0); in pud_clear()
|
/arch/powerpc/mm/ |
D | pgtable.c | 351 pud_t pud, *pudp; in __find_linux_pte() local 394 pudp = pud_offset(&p4d, ea); in __find_linux_pte() 395 pud = READ_ONCE(*pudp); in __find_linux_pte() 401 ret_pte = (pte_t *)pudp; in __find_linux_pte()
|