| /arch/arm/include/asm/ |
| D | hugetlb-3level.h | 32 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
34 pte_t retval = *ptep; in huge_ptep_get()
41 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
43 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
47 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
49 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
53 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
55 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
59 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
61 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
[all …]
|
| /arch/x86/include/asm/ |
| D | hugetlb.h | 39 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
41 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
45 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
47 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
51 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
53 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
67 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
69 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
73 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
76 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| D | pgtable-3level.h | 29 static inline void native_set_pte(pte_t *ptep, pte_t pte) in native_set_pte() argument
31 ptep->pte_high = pte.pte_high; in native_set_pte()
33 ptep->pte_low = pte.pte_low; in native_set_pte()
90 static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) in native_set_pte_atomic() argument
92 set_64bit((unsigned long long *)(ptep), native_pte_val(pte)); in native_set_pte_atomic()
111 pte_t *ptep) in native_pte_clear() argument
113 ptep->pte_low = 0; in native_pte_clear()
115 ptep->pte_high = 0; in native_pte_clear()
143 static inline pte_t native_ptep_get_and_clear(pte_t *ptep) in native_ptep_get_and_clear() argument
147 res.pte = (pteval_t)atomic64_xchg((atomic64_t *)ptep, 0); in native_ptep_get_and_clear()
|
| /arch/ia64/include/asm/ |
| D | hugetlb.h | 24 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
26 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
30 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
32 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
36 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
51 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
53 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
57 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
60 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
63 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/metag/include/asm/ |
| D | hugetlb.h | 26 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
28 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
32 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
34 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
38 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
53 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
55 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
59 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
62 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
65 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/s390/include/asm/ |
| D | hugetlb.h | 20 pte_t *ptep, pte_t pte);
21 pte_t huge_ptep_get(pte_t *ptep);
23 unsigned long addr, pte_t *ptep);
42 pte_t *ptep) in huge_pte_clear() argument
44 pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY; in huge_pte_clear()
48 unsigned long address, pte_t *ptep) in huge_ptep_clear_flush() argument
50 huge_ptep_get_and_clear(vma->vm_mm, address, ptep); in huge_ptep_clear_flush()
54 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
57 int changed = !pte_same(huge_ptep_get(ptep), pte); in huge_ptep_set_access_flags()
59 huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_set_access_flags()
[all …]
|
| D | pgtable.h | 42 #define update_mmu_cache(vma, address, ptep) do { } while (0) argument
43 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0) argument
641 static inline pgste_t pgste_get_lock(pte_t *ptep) in pgste_get_lock() argument
655 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) in pgste_get_lock()
656 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); in pgste_get_lock()
661 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) in pgste_set_unlock() argument
667 : "=Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
668 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
674 static inline pgste_t pgste_get(pte_t *ptep) in pgste_get() argument
678 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); in pgste_get()
[all …]
|
| /arch/mips/include/asm/ |
| D | hugetlb.h | 51 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
53 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
57 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
60 pte_t pte = *ptep; in huge_ptep_get_and_clear()
63 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear()
68 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
75 huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush()
91 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
93 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
98 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument
[all …]
|
| /arch/sparc/include/asm/ |
| D | hugetlb.h | 9 pte_t *ptep, pte_t pte);
12 pte_t *ptep);
43 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
58 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
60 pte_t old_pte = *ptep; in huge_ptep_set_wrprotect()
61 set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); in huge_ptep_set_wrprotect()
65 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
68 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
70 set_huge_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
76 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/sh/include/asm/ |
| D | hugetlb.h | 38 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
40 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
44 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
46 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
50 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
65 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
67 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
71 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
74 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
77 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /arch/tile/include/asm/ |
| D | hugetlb.h | 52 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
54 set_pte(ptep, pte); in set_huge_pte_at()
58 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
60 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
64 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
66 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
80 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
82 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
86 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
89 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| /arch/sparc/mm/ |
| D | hugetlbpage.c | 176 pte_t *ptep, pte_t entry) in set_huge_pte_at() argument
182 if (!pte_present(*ptep) && pte_present(entry)) in set_huge_pte_at()
188 orig[0] = *ptep; in set_huge_pte_at()
189 orig[1] = *(ptep + nptes / 2); in set_huge_pte_at()
191 *ptep = entry; in set_huge_pte_at()
192 ptep++; in set_huge_pte_at()
199 ptep -= nptes / 2; in set_huge_pte_at()
200 maybe_tlb_batch_add(mm, addr, ptep, orig[1], 0); in set_huge_pte_at()
202 ptep -= nptes / 2; in set_huge_pte_at()
203 maybe_tlb_batch_add(mm, addr, ptep, orig[0], 0); in set_huge_pte_at()
[all …]
|
| /arch/parisc/mm/ |
| D | hugetlbpage.c | 115 pte_t *ptep, pte_t entry) in __set_huge_pte_at() argument
124 set_pte(ptep, entry); in __set_huge_pte_at()
125 ptep++; in __set_huge_pte_at()
135 pte_t *ptep, pte_t entry) in set_huge_pte_at() argument
140 __set_huge_pte_at(mm, addr, ptep, entry); in set_huge_pte_at()
146 pte_t *ptep) in huge_ptep_get_and_clear() argument
152 entry = *ptep; in huge_ptep_get_and_clear()
153 __set_huge_pte_at(mm, addr, ptep, __pte(0)); in huge_ptep_get_and_clear()
161 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
167 old_pte = *ptep; in huge_ptep_set_wrprotect()
[all …]
|
| /arch/powerpc/include/asm/ |
| D | hugetlb.h | 116 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
118 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
122 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
125 return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1)); in huge_ptep_get_and_clear()
127 return __pte(pte_update(ptep, ~0UL, 0)); in huge_ptep_get_and_clear()
132 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
135 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush()
150 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
159 ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
162 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| D | pgtable.h | 109 extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
118 pte_t *ptep, pte_t pte, int percpu) in __set_pte_at() argument
128 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
131 pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte)); in __set_pte_at()
143 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
148 if (pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
149 flush_hash_entry(mm, ptep, addr); in __set_pte_at()
155 : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) in __set_pte_at()
164 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) in __set_pte_at()
171 *ptep = pte; in __set_pte_at()
[all …]
|
| D | pgtable-ppc32.h | 127 #define pte_clear(mm, addr, ptep) \ argument
128 do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0)
147 extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
232 static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
235 old = pte_update(ptep, _PAGE_ACCESSED, 0); in __ptep_test_and_clear_young()
238 unsigned long ptephys = __pa(ptep) & PAGE_MASK; in __ptep_test_and_clear_young()
249 pte_t *ptep) in ptep_get_and_clear() argument
251 return __pte(pte_update(ptep, ~_PAGE_HASHPTE, 0)); in ptep_get_and_clear()
256 pte_t *ptep) in ptep_set_wrprotect() argument
258 pte_update(ptep, (_PAGE_RW | _PAGE_HWWRITE), _PAGE_RO); in ptep_set_wrprotect()
[all …]
|
| D | pgtable-ppc64.h | 204 pte_t *ptep, unsigned long pte, int huge);
209 pte_t *ptep, unsigned long clr, in pte_update() argument
224 : "=&r" (old), "=&r" (tmp), "=m" (*ptep) in pte_update()
225 : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set) in pte_update()
228 unsigned long old = pte_val(*ptep); in pte_update()
229 *ptep = __pte((old & ~clr) | set); in pte_update()
237 hpte_need_flush(mm, addr, ptep, old, huge); in pte_update()
244 unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
248 if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) in __ptep_test_and_clear_young()
250 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in __ptep_test_and_clear_young()
[all …]
|
| D | tlb.h | 37 extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
40 static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, in __tlb_remove_tlb_entry() argument
44 if (pte_val(*ptep) & _PAGE_HASHPTE) in __tlb_remove_tlb_entry()
45 flush_hash_entry(tlb->mm, ptep, address); in __tlb_remove_tlb_entry()
|
| /arch/parisc/include/asm/ |
| D | hugetlb.h | 9 pte_t *ptep, pte_t pte);
12 pte_t *ptep);
43 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
58 unsigned long addr, pte_t *ptep);
61 unsigned long addr, pte_t *ptep,
64 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
66 return *ptep; in huge_ptep_get()
|
| /arch/xtensa/include/asm/ |
| D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument
25 (pmd_val(*(pmdp)) = ((unsigned long)ptep))
44 pte_t *ptep; in pte_alloc_one_kernel() local
47 ptep = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); in pte_alloc_one_kernel()
48 if (!ptep) in pte_alloc_one_kernel()
51 pte_clear(NULL, 0, ptep + i); in pte_alloc_one_kernel()
52 return ptep; in pte_alloc_one_kernel()
|
| D | pgtable.h | 252 #define pte_clear(mm,addr,ptep) \ argument
253 do { update_pte(ptep, __pte(_PAGE_CA_INVALID | _PAGE_USER)); } while (0)
303 static inline void update_pte(pte_t *ptep, pte_t pteval) in update_pte() argument
305 *ptep = pteval; in update_pte()
307 __asm__ __volatile__ ("dhwb %0, 0" :: "a" (ptep)); in update_pte()
315 set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval) in set_pte_at() argument
317 update_pte(ptep, pteval); in set_pte_at()
320 static inline void set_pte(pte_t *ptep, pte_t pteval) in set_pte() argument
322 update_pte(ptep, pteval); in set_pte()
335 pte_t *ptep) in ptep_test_and_clear_young() argument
[all …]
|
| /arch/arm64/include/asm/ |
| D | hugetlb.h | 24 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
26 return *ptep; in huge_ptep_get()
75 pte_t *ptep, pte_t pte);
77 unsigned long addr, pte_t *ptep,
80 unsigned long addr, pte_t *ptep);
82 unsigned long addr, pte_t *ptep);
84 unsigned long addr, pte_t *ptep);
|
| /arch/arm64/mm/ |
| D | hugetlbpage.c | 45 pte_t *ptep, pte_t pte, size_t *pgsize) in find_num_contig() argument
68 if ((pte_t *)pmd == ptep) { in find_num_contig()
76 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
80 int ncontig = find_num_contig(mm, addr, ptep, pte, &pgsize); in set_huge_pte_at()
85 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
92 pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, in set_huge_pte_at()
94 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); in set_huge_pte_at()
95 ptep++; in set_huge_pte_at()
201 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
205 if (pte_cont(*ptep)) { in huge_ptep_get_and_clear()
[all …]
|
| /arch/s390/mm/ |
| D | pageattr.c | 48 pte_t *ptep; in walk_page_table() local
59 ptep = pte_offset_kernel(pmdp, addr); in walk_page_table()
60 if (pte_none(*ptep)) in walk_page_table()
62 return ptep; in walk_page_table()
68 pte_t *ptep, pte; in change_page_attr() local
72 ptep = walk_page_table(addr); in change_page_attr()
73 if (WARN_ON_ONCE(!ptep)) in change_page_attr()
75 pte = *ptep; in change_page_attr()
77 __ptep_ipte(addr, ptep); in change_page_attr()
78 *ptep = pte; in change_page_attr()
|
| /arch/x86/xen/ |
| D | p2m.c | 211 pte_t *ptep; in xen_build_mfn_list_list() local
240 ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), in xen_build_mfn_list_list()
242 BUG_ON(!ptep || level != PG_LEVEL_4K); in xen_build_mfn_list_list()
243 mfn = pte_mfn(*ptep); in xen_build_mfn_list_list()
244 ptep = (pte_t *)((unsigned long)ptep & ~(PAGE_SIZE - 1)); in xen_build_mfn_list_list()
250 if (ptep == p2m_missing_pte || ptep == p2m_identity_pte) { in xen_build_mfn_list_list()
334 pte_t *ptep; in xen_rebuild_p2m_list() local
386 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list()
387 set_pte(ptep, in xen_rebuild_p2m_list()
396 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list()
[all …]
|