| /arch/powerpc/mm/ |
| D | pgtable-hash64.c | 64 pmd_t *pmdp; in hash__map_kernel_page() local
73 pmdp = pmd_alloc(&init_mm, pudp, ea); in hash__map_kernel_page()
74 if (!pmdp) in hash__map_kernel_page()
76 ptep = pte_alloc_kernel(pmdp, ea); in hash__map_kernel_page()
103 pmd_t *pmdp, unsigned long clr, in hash__pmd_hugepage_update() argument
110 WARN_ON(!pmd_trans_huge(*pmdp)); in hash__pmd_hugepage_update()
122 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp) in hash__pmd_hugepage_update()
123 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp), in hash__pmd_hugepage_update()
131 hpte_do_hugepage_flush(mm, addr, pmdp, old); in hash__pmd_hugepage_update()
136 pmd_t *pmdp) in hash__pmdp_collapse_flush() argument
[all …]
|
| D | 40x_mmu.c | 109 pmd_t *pmdp; in mmu_mapin_ram() local
112 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); in mmu_mapin_ram()
113 *pmdp++ = __pmd(val); in mmu_mapin_ram()
114 *pmdp++ = __pmd(val); in mmu_mapin_ram()
115 *pmdp++ = __pmd(val); in mmu_mapin_ram()
116 *pmdp++ = __pmd(val); in mmu_mapin_ram()
124 pmd_t *pmdp; in mmu_mapin_ram() local
127 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); in mmu_mapin_ram()
128 *pmdp = __pmd(val); in mmu_mapin_ram()
|
| D | pgtable-book3e.c | 77 pmd_t *pmdp; in map_kernel_page() local
86 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page()
87 if (!pmdp) in map_kernel_page()
89 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
105 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in map_kernel_page()
106 BUG_ON(pmdp == NULL); in map_kernel_page()
107 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page()
109 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
110 if (!pmd_present(*pmdp)) { in map_kernel_page()
113 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
[all …]
|
| D | pgtable-radix.c | 49 pmd_t *pmdp; in radix__map_kernel_page() local
64 pmdp = pmd_alloc(&init_mm, pudp, ea); in radix__map_kernel_page()
65 if (!pmdp) in radix__map_kernel_page()
68 ptep = pmdp_ptep(pmdp); in radix__map_kernel_page()
71 ptep = pte_alloc_kernel(pmdp, ea); in radix__map_kernel_page()
87 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in radix__map_kernel_page()
88 BUG_ON(pmdp == NULL); in radix__map_kernel_page()
89 pud_populate(&init_mm, pudp, pmdp); in radix__map_kernel_page()
91 pmdp = pmd_offset(pudp, ea); in radix__map_kernel_page()
93 ptep = pmdp_ptep(pmdp); in radix__map_kernel_page()
[all …]
|
| D | pgtable-book3s64.c | 29 pmd_t *pmdp, pmd_t entry, int dirty) in pmdp_set_access_flags() argument
33 WARN_ON(!pmd_trans_huge(*pmdp)); in pmdp_set_access_flags()
36 changed = !pmd_same(*(pmdp), entry); in pmdp_set_access_flags()
38 __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp), pmd_pte(entry)); in pmdp_set_access_flags()
45 unsigned long address, pmd_t *pmdp) in pmdp_test_and_clear_young() argument
47 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp); in pmdp_test_and_clear_young()
54 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
57 WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp))); in set_pmd_at()
62 return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd)); in set_pmd_at()
69 pmd_t *pmdp) in pmdp_invalidate() argument
[all …]
|
| D | hugepage-hash64.c | 22 pmd_t *pmdp, unsigned long trap, unsigned long flags, in __hash_page_thp() argument
36 pmd_t pmd = READ_ONCE(*pmdp); in __hash_page_thp()
52 } while (!pmd_xchg(pmdp, __pmd(old_pmd), __pmd(new_pmd))); in __hash_page_thp()
74 hpte_slot_array = get_hpte_slot_array(pmdp); in __hash_page_thp()
82 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, in __hash_page_thp()
160 *pmdp = __pmd(old_pmd); in __hash_page_thp()
184 *pmdp = __pmd(new_pmd & ~H_PAGE_BUSY); in __hash_page_thp()
|
| /arch/s390/mm/ |
| D | pageattr.c | 89 static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, in walk_pte_level() argument
94 ptep = pte_offset(pmdp, addr); in walk_pte_level()
110 static int split_pmd_page(pmd_t *pmdp, unsigned long addr) in split_pmd_page() argument
120 pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; in split_pmd_page()
121 ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); in split_pmd_page()
130 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); in split_pmd_page()
136 static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa) in modify_pmd_page() argument
141 new = pmd_wrprotect(*pmdp); in modify_pmd_page()
143 new = pmd_mkwrite(pmd_mkdirty(*pmdp)); in modify_pmd_page()
144 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); in modify_pmd_page()
[all …]
|
| D | gup.c | 20 static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr, in gup_pte_range() argument
56 static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, in gup_huge_pmd() argument
84 if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { in gup_huge_pmd()
99 pmd_t *pmdp, pmd; in gup_pmd_range() local
101 pmdp = (pmd_t *) pudp; in gup_pmd_range()
103 pmdp = (pmd_t *) pud_deref(pud); in gup_pmd_range()
104 pmdp += pmd_index(addr); in gup_pmd_range()
106 pmd = *pmdp; in gup_pmd_range()
119 if (!gup_huge_pmd(pmdp, pmd, addr, next, in gup_pmd_range()
122 } else if (!gup_pte_range(pmdp, pmd, addr, next, in gup_pmd_range()
[all …]
|
| D | hugetlbpage.c | 143 pmd_t *pmdp = (pmd_t *) ptep; in huge_ptep_get_and_clear() local
149 pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); in huge_ptep_get_and_clear()
158 pmd_t *pmdp = NULL; in huge_pte_alloc() local
166 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc()
168 return (pte_t *) pmdp; in huge_pte_alloc()
175 pmd_t *pmdp = NULL; in huge_pte_offset() local
183 pmdp = pmd_offset(pudp, addr); in huge_pte_offset()
186 return (pte_t *) pmdp; in huge_pte_offset()
|
| D | pgtable.c | 291 unsigned long addr, pmd_t *pmdp) in pmdp_flush_direct() argument
295 old = *pmdp; in pmdp_flush_direct()
299 __pmdp_csp(pmdp); in pmdp_flush_direct()
305 __pmdp_idte(addr, pmdp, IDTE_LOCAL); in pmdp_flush_direct()
307 __pmdp_idte(addr, pmdp, IDTE_GLOBAL); in pmdp_flush_direct()
313 unsigned long addr, pmd_t *pmdp) in pmdp_flush_lazy() argument
317 old = *pmdp; in pmdp_flush_lazy()
323 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; in pmdp_flush_lazy()
326 __pmdp_idte(addr, pmdp, IDTE_GLOBAL); in pmdp_flush_lazy()
328 __pmdp_csp(pmdp); in pmdp_flush_lazy()
[all …]
|
| /arch/powerpc/include/asm/book3s/32/ |
| D | pgalloc.h | 25 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
28 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel()
31 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
34 *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_PRESENT); in pmd_populate()
40 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
43 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel()
46 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
49 *pmdp = __pmd((unsigned long)lowmem_page_address(pte_page) | _PMD_PRESENT); in pmd_populate()
|
| /arch/powerpc/include/asm/nohash/32/ |
| D | pgalloc.h | 25 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
28 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel()
31 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
34 *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_PRESENT); in pmd_populate()
40 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
43 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel()
46 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
49 *pmdp = __pmd((unsigned long)lowmem_page_address(pte_page) | _PMD_PRESENT); in pmd_populate()
|
| /arch/powerpc/include/asm/book3s/64/ |
| D | hash-4k.h | 55 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument
98 unsigned long addr, pmd_t *pmdp,
101 unsigned long address, pmd_t *pmdp);
102 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
104 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
106 unsigned long address, pmd_t *pmdp);
108 unsigned long addr, pmd_t *pmdp);
|
| D | hash-64k.h | 122 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument
131 return *(char **)(pmdp + PTRS_PER_PMD); in get_hpte_slot_array()
197 unsigned long addr, pmd_t *pmdp,
200 unsigned long address, pmd_t *pmdp);
201 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
203 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
205 unsigned long address, pmd_t *pmdp);
207 unsigned long addr, pmd_t *pmdp);
|
| D | pgtable.h | 641 static inline void pmd_set(pmd_t *pmdp, unsigned long val) in pmd_set() argument
643 *pmdp = __pmd(val); in pmd_set()
646 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
648 *pmdp = __pmd(0); in pmd_clear()
873 pmd_t *pmdp, pmd_t pmd);
886 pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, in pmd_hugepage_update() argument
890 return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set); in pmd_hugepage_update()
891 return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set); in pmd_hugepage_update()
908 unsigned long addr, pmd_t *pmdp) in __pmdp_test_and_clear_young() argument
912 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) in __pmdp_test_and_clear_young()
[all …]
|
| /arch/unicore32/include/asm/ |
| D | pgalloc.h | 82 static inline void __pmd_populate(pmd_t *pmdp, unsigned long pmdval) in __pmd_populate() argument
84 set_pmd(pmdp, __pmd(pmdval)); in __pmd_populate()
85 flush_pmd_entry(pmdp); in __pmd_populate()
93 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
101 __pmd_populate(pmdp, __pa(pte_ptr) | _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
105 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
107 __pmd_populate(pmdp, in pmd_populate()
|
| /arch/sparc/mm/ |
| D | tlb.c | 167 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
169 pmd_t orig = *pmdp; in set_pmd_at()
171 *pmdp = pmd; in set_pmd_at()
226 pmd_t *pmdp) in pmdp_invalidate() argument
228 pmd_t entry = *pmdp; in pmdp_invalidate()
232 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_invalidate()
245 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
253 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
256 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
257 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
[all …]
|
| /arch/arm/include/asm/ |
| D | pgalloc.h | 131 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
135 pmdp[0] = __pmd(pmdval); in __pmd_populate()
137 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); in __pmd_populate()
139 flush_pmd_entry(pmdp); in __pmd_populate()
149 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
154 __pmd_populate(pmdp, __pa(ptep), _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
158 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
168 __pmd_populate(pmdp, page_to_phys(ptep), prot); in pmd_populate()
|
| /arch/m68k/include/asm/ |
| D | motorola_pgtable.h | 108 static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
111 unsigned long *ptr = pmdp->pmd; in pmd_set()
119 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
121 pgd_val(*pgdp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pgd_set()
140 #define pmd_clear(pmdp) ({ \ argument
141 unsigned long *__ptr = pmdp->pmd; \
217 static inline pte_t *pte_offset_kernel(pmd_t *pmdp, unsigned long address) in pte_offset_kernel() argument
219 return (pte_t *)__pmd_page(*pmdp) + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); in pte_offset_kernel()
222 #define pte_offset_map(pmdp,address) ((pte_t *)__pmd_page(*pmdp) + (((address) >> PAGE_SHIFT) & (PT… argument
241 pmd_t *pmdp; in nocache_page() local
[all …]
|
| D | mcf_pgtable.h | 164 #define pmd_set(pmdp, ptep) do {} while (0) argument
166 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
168 pgd_val(*pgdp) = virt_to_phys(pmdp); in pgd_set()
198 static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; } in pmd_clear() argument
362 pmd_t *pmdp; in nocache_page() local
367 pmdp = pmd_offset(dir, addr); in nocache_page()
368 ptep = pte_offset_kernel(pmdp, addr); in nocache_page()
378 pmd_t *pmdp; in cache_page() local
383 pmdp = pmd_offset(dir, addr); in cache_page()
384 ptep = pte_offset_kernel(pmdp, addr); in cache_page()
[all …]
|
| /arch/tile/include/asm/ |
| D | pgalloc.h | 41 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) in set_pmd() argument
44 set_pte(pmdp, pmd); in set_pmd()
46 set_pte(&pmdp->pud.pgd, pmd.pud.pgd); in set_pmd()
150 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp) in pmd_free() argument
152 pgtable_free(mm, virt_to_page(pmdp), L1_USER_PGTABLE_ORDER); in pmd_free()
155 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, in __pmd_free_tlb() argument
158 __pgtable_free_tlb(tlb, virt_to_page(pmdp), address, in __pmd_free_tlb()
|
| /arch/arm64/include/asm/ |
| D | pgalloc.h | 129 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
132 set_pmd(pmdp, __pmd(pte | prot)); in __pmd_populate()
140 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
145 __pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE); in pmd_populate_kernel()
149 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
151 __pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE); in pmd_populate()
|
| /arch/x86/include/asm/ |
| D | pgtable-3level.h | 70 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) in pmd_read_atomic() argument
73 u32 *tmp = (u32 *)pmdp; in pmd_read_atomic()
93 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
95 set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd)); in native_set_pmd()
164 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp) in native_pmdp_get_and_clear() argument
166 union split_pmd res, *orig = (union split_pmd *)pmdp; in native_pmdp_get_and_clear()
|
| /arch/s390/include/asm/ |
| D | pgtable.h | 785 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
787 pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID; in pmd_clear()
1234 static inline void __pmdp_csp(pmd_t *pmdp) in __pmdp_csp() argument
1236 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), in __pmdp_csp()
1237 pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); in __pmdp_csp()
1243 static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp, int local) in __pmdp_idte() argument
1247 sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); in __pmdp_idte()
1250 : "+m" (*pmdp) in __pmdp_idte()
1277 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1281 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
[all …]
|
| /arch/xtensa/include/asm/ |
| D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument
25 (pmd_val(*(pmdp)) = ((unsigned long)ptep))
26 #define pmd_populate(mm, pmdp, page) \ argument
27 (pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page)))
|