| /mm/kasan/ |
| D | init.c | 45 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument
47 return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); in kasan_pud_table()
50 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument
142 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument
145 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate()
182 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate() local
191 p4d_populate(&init_mm, p4d, in zero_p4d_populate()
193 pud = pud_offset(p4d, addr); in zero_p4d_populate()
202 if (p4d_none(*p4d)) { in zero_p4d_populate()
206 p = pud_alloc(&init_mm, p4d, addr); in zero_p4d_populate()
[all …]
|
| D | shadow.c | 143 p4d_t *p4d; in shadow_mapped() local
150 p4d = p4d_offset(pgd, addr); in shadow_mapped()
151 if (p4d_none(*p4d)) in shadow_mapped()
153 pud = pud_offset(p4d, addr); in shadow_mapped()
|
| /mm/ |
| D | ioremap.c | 152 static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, in ioremap_pud_range() argument
159 pud = pud_alloc_track(&init_mm, p4d, addr, mask); in ioremap_pud_range()
176 static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, in ioremap_try_huge_p4d() argument
192 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) in ioremap_try_huge_p4d()
195 return p4d_set_huge(p4d, phys_addr, prot); in ioremap_try_huge_p4d()
202 p4d_t *p4d; in ioremap_p4d_range() local
205 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); in ioremap_p4d_range()
206 if (!p4d) in ioremap_p4d_range()
211 if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) { in ioremap_p4d_range()
216 if (ioremap_pud_range(p4d, addr, next, phys_addr, prot, mask)) in ioremap_p4d_range()
[all …]
|
| D | sparse-vmemmap.c | 183 pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) in vmemmap_pud_populate() argument
185 pud_t *pud = pud_offset(p4d, addr); in vmemmap_pud_populate()
197 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate() local
198 if (p4d_none(*p4d)) { in vmemmap_p4d_populate()
202 p4d_populate(&init_mm, p4d, p); in vmemmap_p4d_populate()
204 return p4d; in vmemmap_p4d_populate()
224 p4d_t *p4d; in vmemmap_populate_basepages() local
233 p4d = vmemmap_p4d_populate(pgd, addr, node); in vmemmap_populate_basepages()
234 if (!p4d) in vmemmap_populate_basepages()
236 pud = vmemmap_pud_populate(p4d, addr, node); in vmemmap_populate_basepages()
|
| D | pgalloc-track.h | 19 static inline pud_t *pud_alloc_track(struct mm_struct *mm, p4d_t *p4d, in pud_alloc_track() argument
23 if (unlikely(p4d_none(*p4d))) { in pud_alloc_track()
24 if (__pud_alloc(mm, p4d, address)) in pud_alloc_track()
29 return pud_offset(p4d, address); in pud_alloc_track()
|
| D | memory.c | 298 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, in free_pud_range() argument
307 pud = pud_offset(p4d, addr); in free_pud_range()
326 pud = pud_offset(p4d, start); in free_pud_range()
327 p4d_clear(p4d); in free_pud_range()
336 p4d_t *p4d; in free_p4d_range() local
341 p4d = p4d_offset(pgd, addr); in free_p4d_range()
344 if (p4d_none_or_clear_bad(p4d)) in free_p4d_range()
346 free_pud_range(tlb, p4d, addr, next, floor, ceiling); in free_p4d_range()
347 } while (p4d++, addr = next, addr != end); in free_p4d_range()
360 p4d = p4d_offset(pgd, start); in free_p4d_range()
[all …]
|
| D | pagewalk.c | 119 static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, in walk_pud_range() argument
128 pud = pud_offset(p4d, addr); in walk_pud_range()
171 p4d_t *p4d; in walk_p4d_range() local
177 p4d = p4d_offset(pgd, addr); in walk_p4d_range()
180 if (p4d_none_or_clear_bad(p4d)) { in walk_p4d_range()
188 err = ops->p4d_entry(p4d, addr, next, walk); in walk_p4d_range()
193 err = walk_pud_range(p4d, addr, next, walk); in walk_p4d_range()
196 } while (p4d++, addr = next, addr != end); in walk_p4d_range()
|
| D | debug_vm_pgtable.c | 465 p4d_t p4d; in p4d_basic_tests() local
468 memset(&p4d, RANDOM_NZVALUE, sizeof(p4d_t)); in p4d_basic_tests()
469 WARN_ON(!p4d_same(p4d, p4d)); in p4d_basic_tests()
525 p4d_t p4d = READ_ONCE(*p4dp); in p4d_clear_tests() local
531 p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE); in p4d_clear_tests()
532 WRITE_ONCE(*p4dp, p4d); in p4d_clear_tests()
534 p4d = READ_ONCE(*p4dp); in p4d_clear_tests()
535 WARN_ON(!p4d_none(p4d)); in p4d_clear_tests()
541 p4d_t p4d; in p4d_populate_tests() local
554 p4d = READ_ONCE(*p4dp); in p4d_populate_tests()
[all …]
|
| D | gup.c | 747 p4d_t *p4d; in follow_p4d_mask() local
750 p4d = p4d_offset(pgdp, address); in follow_p4d_mask()
751 if (p4d_none(*p4d)) in follow_p4d_mask()
753 BUILD_BUG_ON(p4d_huge(*p4d)); in follow_p4d_mask()
754 if (unlikely(p4d_bad(*p4d))) in follow_p4d_mask()
757 if (is_hugepd(__hugepd(p4d_val(*p4d)))) { in follow_p4d_mask()
759 __hugepd(p4d_val(*p4d)), flags, in follow_p4d_mask()
765 return follow_pud_mask(vma, address, p4d, flags, ctx); in follow_p4d_mask()
844 p4d_t *p4d; in get_gate_page() local
859 p4d = p4d_offset(pgd, address); in get_gate_page()
[all …]
|
| D | mprotect.c | 285 p4d_t *p4d, unsigned long addr, unsigned long end, in change_pud_range() argument
292 pud = pud_offset(p4d, addr); in change_pud_range()
308 p4d_t *p4d; in change_p4d_range() local
312 p4d = p4d_offset(pgd, addr); in change_p4d_range()
315 if (p4d_none_or_clear_bad(p4d)) in change_p4d_range()
317 pages += change_pud_range(vma, p4d, addr, next, newprot, in change_p4d_range()
319 } while (p4d++, addr = next, addr != end); in change_p4d_range()
|
| D | page_vma_mapped.c | 155 p4d_t *p4d; in page_vma_mapped_walk() local
198 p4d = p4d_offset(pgd, pvmw->address); in page_vma_mapped_walk()
199 if (!p4d_present(*p4d)) { in page_vma_mapped_walk()
203 pud = pud_offset(p4d, pvmw->address); in page_vma_mapped_walk()
|
| D | mremap.c | 36 p4d_t *p4d; in get_old_pud() local
43 p4d = p4d_offset(pgd, addr); in get_old_pud()
44 if (p4d_none_or_clear_bad(p4d)) in get_old_pud()
47 pud = pud_offset(p4d, addr); in get_old_pud()
74 p4d_t *p4d; in alloc_new_pud() local
77 p4d = p4d_alloc(mm, pgd, addr); in alloc_new_pud()
78 if (!p4d) in alloc_new_pud()
81 return pud_alloc(mm, p4d, addr); in alloc_new_pud()
|
| D | vmalloc.c | 111 static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, in vunmap_pud_range() argument
118 pud = pud_offset(p4d, addr); in vunmap_pud_range()
137 p4d_t *p4d; in vunmap_p4d_range() local
141 p4d = p4d_offset(pgd, addr); in vunmap_p4d_range()
145 cleared = p4d_clear_huge(p4d); in vunmap_p4d_range()
146 if (cleared || p4d_bad(*p4d)) in vunmap_p4d_range()
151 if (p4d_none_or_clear_bad(p4d)) in vunmap_p4d_range()
153 vunmap_pud_range(p4d, addr, next, mask); in vunmap_p4d_range()
154 } while (p4d++, addr = next, addr != end); in vunmap_p4d_range()
239 static int vmap_pud_range(p4d_t *p4d, unsigned long addr, in vmap_pud_range() argument
[all …]
|
| D | pgtable-generic.c | 28 void p4d_clear_bad(p4d_t *p4d) in p4d_clear_bad() argument
30 p4d_ERROR(*p4d); in p4d_clear_bad()
31 p4d_clear(p4d); in p4d_clear_bad()
|
| D | userfaultfd.c | 256 p4d_t *p4d; in mm_alloc_pmd() local
260 p4d = p4d_alloc(mm, pgd, address); in mm_alloc_pmd()
261 if (!p4d) in mm_alloc_pmd()
263 pud = pud_alloc(mm, p4d, address); in mm_alloc_pmd()
|
| D | ptdump.c | 49 static int ptdump_p4d_entry(p4d_t *p4d, unsigned long addr, in ptdump_p4d_entry() argument
53 p4d_t val = READ_ONCE(*p4d); in ptdump_p4d_entry()
|
| D | memory-failure.c | 297 p4d_t *p4d; in dev_pagemap_mapping_shift() local
305 p4d = p4d_offset(pgd, address); in dev_pagemap_mapping_shift()
306 if (!p4d_present(*p4d)) in dev_pagemap_mapping_shift()
308 pud = pud_offset(p4d, address); in dev_pagemap_mapping_shift()
|
| D | rmap.c | 744 p4d_t *p4d; in mm_find_pmd() local
753 p4d = p4d_offset(pgd, address); in mm_find_pmd()
754 if (!p4d_present(*p4d)) in mm_find_pmd()
757 pud = pud_offset(p4d, address); in mm_find_pmd()
|
| D | hugetlb.c | 5507 p4d_t *p4d = p4d_offset(pgd, *addr); in huge_pmd_unshare() local
5508 pud_t *pud = pud_offset(p4d, *addr); in huge_pmd_unshare()
5558 p4d_t *p4d; in huge_pte_alloc() local
5563 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
5564 if (!p4d) in huge_pte_alloc()
5566 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc()
5596 p4d_t *p4d; in huge_pte_offset() local
5603 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
5604 if (!p4d_present(*p4d)) in huge_pte_offset()
5607 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
| D | swapfile.c | 2075 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, in unuse_pud_range() argument
2084 pud = pud_offset(p4d, addr); in unuse_pud_range()
2102 p4d_t *p4d; in unuse_p4d_range() local
2106 p4d = p4d_offset(pgd, addr); in unuse_p4d_range()
2109 if (p4d_none_or_clear_bad(p4d)) in unuse_p4d_range()
2111 ret = unuse_pud_range(vma, p4d, addr, next, type, in unuse_p4d_range()
2115 } while (p4d++, addr = next, addr != end); in unuse_p4d_range()
|
| D | huge_memory.c | 2305 p4d_t *p4d; in split_huge_pmd_address() local
2313 p4d = p4d_offset(pgd, address); in split_huge_pmd_address()
2314 if (!p4d_present(*p4d)) in split_huge_pmd_address()
2317 pud = pud_offset(p4d, address); in split_huge_pmd_address()
|