| /arch/s390/mm/ |
| D | pgtable.c | 29 pte_t *ptep, int nodat) in ptep_ipte_local() argument
42 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); in ptep_ipte_local()
44 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local()
49 pte_t *ptep, int nodat) in ptep_ipte_global() argument
62 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); in ptep_ipte_global()
64 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global()
69 unsigned long addr, pte_t *ptep, in ptep_flush_direct() argument
74 old = *ptep; in ptep_flush_direct()
80 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
82 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
[all …]
|
| /arch/arm/include/asm/ |
| D | hugetlb-3level.h | 32 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
34 pte_t retval = *ptep; in huge_ptep_get()
41 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
43 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
47 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
49 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
53 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
55 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
59 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
61 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
[all …]
|
| /arch/arm64/mm/ |
| D | hugetlbpage.c | 55 pte_t *ptep, size_t *pgsize) in find_num_contig() argument
64 if ((pte_t *)pmd == ptep) { in find_num_contig()
107 pte_t *ptep, in get_clear_flush() argument
112 pte_t orig_pte = huge_ptep_get(ptep); in get_clear_flush()
116 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { in get_clear_flush()
117 pte_t pte = ptep_get_and_clear(mm, addr, ptep); in get_clear_flush()
147 pte_t *ptep, in clear_flush() argument
154 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) in clear_flush()
155 pte_clear(mm, addr, ptep); in clear_flush()
161 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
[all …]
|
| /arch/s390/include/asm/ |
| D | hugetlb.h | 21 pte_t *ptep, pte_t pte);
22 pte_t huge_ptep_get(pte_t *ptep);
24 unsigned long addr, pte_t *ptep);
43 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument
45 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) in huge_pte_clear()
46 pte_val(*ptep) = _REGION3_ENTRY_EMPTY; in huge_pte_clear()
48 pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY; in huge_pte_clear()
52 unsigned long address, pte_t *ptep) in huge_ptep_clear_flush() argument
54 huge_ptep_get_and_clear(vma->vm_mm, address, ptep); in huge_ptep_clear_flush()
58 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
[all …]
|
| /arch/ia64/include/asm/ |
| D | hugetlb.h | 25 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
27 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
31 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
33 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
37 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
52 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
54 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
58 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
61 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
64 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/metag/include/asm/ |
| D | hugetlb.h | 27 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
29 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
33 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
35 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
39 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
54 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
56 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
60 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
63 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
66 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/x86/include/asm/ |
| D | hugetlb.h | 40 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
42 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
46 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
48 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
52 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
54 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
68 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
70 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
74 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
77 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| D | pgtable-3level.h | 30 static inline void native_set_pte(pte_t *ptep, pte_t pte) in native_set_pte() argument
32 ptep->pte_high = pte.pte_high; in native_set_pte()
34 ptep->pte_low = pte.pte_low; in native_set_pte()
91 static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) in native_set_pte_atomic() argument
93 set_64bit((unsigned long long *)(ptep), native_pte_val(pte)); in native_set_pte_atomic()
112 pte_t *ptep) in native_pte_clear() argument
114 ptep->pte_low = 0; in native_pte_clear()
116 ptep->pte_high = 0; in native_pte_clear()
148 static inline pte_t native_ptep_get_and_clear(pte_t *ptep) in native_ptep_get_and_clear() argument
152 res.pte = (pteval_t)atomic64_xchg((atomic64_t *)ptep, 0); in native_ptep_get_and_clear()
[all …]
|
| /arch/sh/include/asm/ |
| D | hugetlb.h | 39 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
41 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
45 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
47 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
51 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
66 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
68 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
72 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
75 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
78 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/mips/include/asm/ |
| D | hugetlb.h | 51 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
53 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
57 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
60 pte_t pte = *ptep; in huge_ptep_get_and_clear()
63 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear()
68 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
85 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
87 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
92 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument
95 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
[all …]
|
| /arch/sparc/include/asm/ |
| D | hugetlb.h | 17 pte_t *ptep, pte_t pte);
20 pte_t *ptep);
45 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
60 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
62 pte_t old_pte = *ptep; in huge_ptep_set_wrprotect()
63 set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); in huge_ptep_set_wrprotect()
67 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
70 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
72 set_huge_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
78 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/tile/include/asm/ |
| D | hugetlb.h | 52 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
54 set_pte(ptep, pte); in set_huge_pte_at()
58 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
60 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
64 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
66 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
80 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
82 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
86 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
89 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| /arch/powerpc/include/asm/nohash/64/ |
| D | pgtable.h | 176 pte_t *ptep, unsigned long pte, int huge);
181 pte_t *ptep, unsigned long clr, in pte_update() argument
196 : "=&r" (old), "=&r" (tmp), "=m" (*ptep) in pte_update()
197 : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set) in pte_update()
200 unsigned long old = pte_val(*ptep); in pte_update()
201 *ptep = __pte((old & ~clr) | set); in pte_update()
209 hpte_need_flush(mm, addr, ptep, old, huge); in pte_update()
216 unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
220 if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) in __ptep_test_and_clear_young()
222 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in __ptep_test_and_clear_young()
[all …]
|
| /arch/parisc/mm/ |
| D | hugetlbpage.c | 118 pte_t *ptep, pte_t entry) in __set_huge_pte_at() argument
127 set_pte(ptep, entry); in __set_huge_pte_at()
128 ptep++; in __set_huge_pte_at()
138 pte_t *ptep, pte_t entry) in set_huge_pte_at() argument
143 __set_huge_pte_at(mm, addr, ptep, entry); in set_huge_pte_at()
149 pte_t *ptep) in huge_ptep_get_and_clear() argument
155 entry = *ptep; in huge_ptep_get_and_clear()
156 __set_huge_pte_at(mm, addr, ptep, __pte(0)); in huge_ptep_get_and_clear()
164 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
170 old_pte = *ptep; in huge_ptep_set_wrprotect()
[all …]
|
| /arch/powerpc/include/asm/book3s/64/ |
| D | radix.h | 127 static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr, in __radix_pte_update() argument
134 pte = READ_ONCE(*ptep); in __radix_pte_update()
138 } while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); in __radix_pte_update()
146 pte_t *ptep, unsigned long clr, in radix__pte_update() argument
156 old_pte = __radix_pte_update(ptep, ~0ul, 0); in radix__pte_update()
163 __radix_pte_update(ptep, 0, new_pte); in radix__pte_update()
165 old_pte = __radix_pte_update(ptep, clr, set); in radix__pte_update()
174 pte_t *ptep, int full) in radix__ptep_get_and_clear_full() argument
186 old_pte = __radix_pte_update(ptep, ~0ul, 0); in radix__ptep_get_and_clear_full()
188 old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0); in radix__ptep_get_and_clear_full()
[all …]
|
| D | hash.h | 101 pte_t *ptep, unsigned long pte, int huge);
106 pte_t *ptep, unsigned long clr, in hash__pte_update() argument
121 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in hash__pte_update()
122 : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), in hash__pte_update()
131 hpte_need_flush(mm, addr, ptep, old, huge); in hash__pte_update()
139 static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) in hash__ptep_set_access_flags() argument
155 :"=&r" (old), "=&r" (tmp), "=m" (*ptep) in hash__ptep_set_access_flags()
156 :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) in hash__ptep_set_access_flags()
176 pte_t *ptep, pte_t pte, int percpu) in hash__set_pte_at() argument
182 *ptep = pte; in hash__set_pte_at()
|
| /arch/arm64/include/asm/ |
| D | hugetlb.h | 23 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
25 return *ptep; in huge_ptep_get()
74 pte_t *ptep, pte_t pte);
76 unsigned long addr, pte_t *ptep,
79 unsigned long addr, pte_t *ptep);
81 unsigned long addr, pte_t *ptep);
83 unsigned long addr, pte_t *ptep);
85 pte_t *ptep, unsigned long sz);
88 pte_t *ptep, pte_t pte, unsigned long sz);
|
| /arch/powerpc/include/asm/book3s/32/ |
| D | pgtable.h | 113 #define pte_clear(mm, addr, ptep) \ argument
114 do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0)
137 extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
204 static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
207 old = pte_update(ptep, _PAGE_ACCESSED, 0); in __ptep_test_and_clear_young()
209 unsigned long ptephys = __pa(ptep) & PAGE_MASK; in __ptep_test_and_clear_young()
219 pte_t *ptep) in ptep_get_and_clear() argument
221 return __pte(pte_update(ptep, ~_PAGE_HASHPTE, 0)); in ptep_get_and_clear()
226 pte_t *ptep) in ptep_set_wrprotect() argument
228 pte_update(ptep, (_PAGE_RW | _PAGE_HWWRITE), _PAGE_RO); in ptep_set_wrprotect()
[all …]
|
| /arch/powerpc/kvm/ |
| D | book3s_64_mmu_radix.c | 169 unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep, in kvmppc_radix_update_pte() argument
176 pte_present(*ptep)) { in kvmppc_radix_update_pte()
178 old = __radix_pte_update(ptep, _PAGE_PRESENT, 0); in kvmppc_radix_update_pte()
183 return __radix_pte_update(ptep, clr, set) | old; in kvmppc_radix_update_pte()
187 pte_t *ptep, pte_t pte) in kvmppc_radix_set_pte_at() argument
189 radix__set_pte_at(kvm->mm, addr, ptep, pte, 0); in kvmppc_radix_set_pte_at()
199 static void kvmppc_pte_free(pte_t *ptep) in kvmppc_pte_free() argument
201 kmem_cache_free(kvm_pte_cache, ptep); in kvmppc_pte_free()
216 pte_t *ptep, *new_ptep = NULL; in kvmppc_create_pte() local
298 ptep = pte_offset_kernel(pmd, gpa); in kvmppc_create_pte()
[all …]
|
| /arch/parisc/include/asm/ |
| D | hugetlb.h | 10 pte_t *ptep, pte_t pte);
13 pte_t *ptep);
44 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
59 unsigned long addr, pte_t *ptep);
62 unsigned long addr, pte_t *ptep,
65 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
67 return *ptep; in huge_ptep_get()
|
| /arch/powerpc/include/asm/nohash/ |
| D | pgtable.h | 117 extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
126 pte_t *ptep, pte_t pte, int percpu) in __set_pte_at() argument
136 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
139 pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte)); in __set_pte_at()
151 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
156 if (pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
157 flush_hash_entry(mm, ptep, addr); in __set_pte_at()
163 : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) in __set_pte_at()
172 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
179 *ptep = pte; in __set_pte_at()
[all …]
|
| /arch/powerpc/mm/ |
| D | pgtable-book3e.c | 78 pte_t *ptep; in map_kernel_page() local
89 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
90 if (!ptep) in map_kernel_page()
92 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, in map_kernel_page()
111 ptep = early_alloc_pgtable(PAGE_SIZE); in map_kernel_page()
112 BUG_ON(ptep == NULL); in map_kernel_page()
113 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
115 ptep = pte_offset_kernel(pmdp, ea); in map_kernel_page()
116 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, in map_kernel_page()
|
| /arch/powerpc/include/asm/ |
| D | hugetlb.h | 149 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
151 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
155 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
158 return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1)); in huge_ptep_get_and_clear()
160 return __pte(pte_update(ptep, ~0UL, 0)); in huge_ptep_get_and_clear()
165 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
168 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush()
183 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
192 ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
195 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| /arch/xtensa/include/asm/ |
| D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument
25 (pmd_val(*(pmdp)) = ((unsigned long)ptep))
44 pte_t *ptep; in pte_alloc_one_kernel() local
47 ptep = (pte_t *)__get_free_page(GFP_KERNEL); in pte_alloc_one_kernel()
48 if (!ptep) in pte_alloc_one_kernel()
51 pte_clear(NULL, 0, ptep + i); in pte_alloc_one_kernel()
52 return ptep; in pte_alloc_one_kernel()
|
| /arch/powerpc/include/asm/nohash/32/ |
| D | pgtable.h | 134 #define pte_clear(mm, addr, ptep) \ argument
135 do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0)
159 extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
244 static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
247 old = pte_update(ptep, _PAGE_ACCESSED, 0); in __ptep_test_and_clear_young()
250 unsigned long ptephys = __pa(ptep) & PAGE_MASK; in __ptep_test_and_clear_young()
261 pte_t *ptep) in ptep_get_and_clear() argument
263 return __pte(pte_update(ptep, ~_PAGE_HASHPTE, 0)); in ptep_get_and_clear()
268 pte_t *ptep) in ptep_set_wrprotect() argument
270 pte_update(ptep, (_PAGE_RW | _PAGE_HWWRITE), _PAGE_RO); in ptep_set_wrprotect()
[all …]
|