| /mm/kasan/ |
| D | kasan_init.c | 88 static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument
91 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate()
116 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate() local
125 p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud)); in zero_p4d_populate()
126 pud = pud_offset(p4d, addr); in zero_p4d_populate()
134 if (p4d_none(*p4d)) { in zero_p4d_populate()
135 p4d_populate(&init_mm, p4d, in zero_p4d_populate()
138 zero_pud_populate(p4d, addr, next); in zero_p4d_populate()
139 } while (p4d++, addr = next, addr != end); in zero_p4d_populate()
160 p4d_t *p4d; in kasan_populate_zero_shadow() local
[all …]
|
| D | kasan.c | 744 p4d_t *p4d; in shadow_mapped() local
751 p4d = p4d_offset(pgd, addr); in shadow_mapped()
752 if (p4d_none(*p4d)) in shadow_mapped()
754 pud = pud_offset(p4d, addr); in shadow_mapped()
|
| /mm/ |
| D | sparse-vmemmap.c | 195 pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) in vmemmap_pud_populate() argument
197 pud_t *pud = pud_offset(p4d, addr); in vmemmap_pud_populate()
209 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate() local
210 if (p4d_none(*p4d)) { in vmemmap_p4d_populate()
214 p4d_populate(&init_mm, p4d, p); in vmemmap_p4d_populate()
216 return p4d; in vmemmap_p4d_populate()
236 p4d_t *p4d; in vmemmap_populate_basepages() local
245 p4d = vmemmap_p4d_populate(pgd, addr, node); in vmemmap_populate_basepages()
246 if (!p4d) in vmemmap_populate_basepages()
248 pud = vmemmap_pud_populate(p4d, addr, node); in vmemmap_populate_basepages()
|
| D | memory.c | 487 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, in free_pud_range() argument
496 pud = pud_offset(p4d, addr); in free_pud_range()
515 pud = pud_offset(p4d, start); in free_pud_range()
516 p4d_clear(p4d); in free_pud_range()
524 p4d_t *p4d; in free_p4d_range() local
529 p4d = p4d_offset(pgd, addr); in free_p4d_range()
532 if (p4d_none_or_clear_bad(p4d)) in free_p4d_range()
534 free_pud_range(tlb, p4d, addr, next, floor, ceiling); in free_p4d_range()
535 } while (p4d++, addr = next, addr != end); in free_p4d_range()
548 p4d = p4d_offset(pgd, start); in free_p4d_range()
[all …]
|
| D | gup.c | 353 p4d_t *p4d; in follow_p4d_mask() local
356 p4d = p4d_offset(pgdp, address); in follow_p4d_mask()
357 if (p4d_none(*p4d)) in follow_p4d_mask()
359 BUILD_BUG_ON(p4d_huge(*p4d)); in follow_p4d_mask()
360 if (unlikely(p4d_bad(*p4d))) in follow_p4d_mask()
363 if (is_hugepd(__hugepd(p4d_val(*p4d)))) { in follow_p4d_mask()
365 __hugepd(p4d_val(*p4d)), flags, in follow_p4d_mask()
371 return follow_pud_mask(vma, address, p4d, flags, page_mask); in follow_p4d_mask()
432 p4d_t *p4d; in get_gate_page() local
447 p4d = p4d_offset(pgd, address); in get_gate_page()
[all …]
|
| D | pagewalk.c | 73 static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, in walk_pud_range() argument
80 pud = pud_offset(p4d, addr); in walk_pud_range()
120 p4d_t *p4d; in walk_p4d_range() local
124 p4d = p4d_offset(pgd, addr); in walk_p4d_range()
127 if (p4d_none_or_clear_bad(p4d)) { in walk_p4d_range()
135 err = walk_pud_range(p4d, addr, next, walk); in walk_p4d_range()
138 } while (p4d++, addr = next, addr != end); in walk_p4d_range()
|
| D | mremap.c | 36 p4d_t *p4d; in get_old_pmd() local
44 p4d = p4d_offset(pgd, addr); in get_old_pmd()
45 if (p4d_none_or_clear_bad(p4d)) in get_old_pmd()
48 pud = pud_offset(p4d, addr); in get_old_pmd()
63 p4d_t *p4d; in alloc_new_pmd() local
68 p4d = p4d_alloc(mm, pgd, addr); in alloc_new_pmd()
69 if (!p4d) in alloc_new_pmd()
71 pud = pud_alloc(mm, p4d, addr); in alloc_new_pmd()
|
| D | mprotect.c | 246 p4d_t *p4d, unsigned long addr, unsigned long end, in change_pud_range() argument
253 pud = pud_offset(p4d, addr); in change_pud_range()
269 p4d_t *p4d; in change_p4d_range() local
273 p4d = p4d_offset(pgd, addr); in change_p4d_range()
276 if (p4d_none_or_clear_bad(p4d)) in change_p4d_range()
278 pages += change_pud_range(vma, p4d, addr, next, newprot, in change_p4d_range()
280 } while (p4d++, addr = next, addr != end); in change_p4d_range()
|
| D | vmalloc.c | 87 static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end) in vunmap_pud_range() argument
92 pud = pud_offset(p4d, addr); in vunmap_pud_range()
105 p4d_t *p4d; in vunmap_p4d_range() local
108 p4d = p4d_offset(pgd, addr); in vunmap_p4d_range()
111 if (p4d_clear_huge(p4d)) in vunmap_p4d_range()
113 if (p4d_none_or_clear_bad(p4d)) in vunmap_p4d_range()
115 vunmap_pud_range(p4d, addr, next); in vunmap_p4d_range()
116 } while (p4d++, addr = next, addr != end); in vunmap_p4d_range()
177 static int vmap_pud_range(p4d_t *p4d, unsigned long addr, in vmap_pud_range() argument
183 pud = pud_alloc(&init_mm, p4d, addr); in vmap_pud_range()
[all …]
|
| D | page_vma_mapped.c | 142 p4d_t *p4d; in page_vma_mapped_walk() local
170 p4d = p4d_offset(pgd, pvmw->address); in page_vma_mapped_walk()
171 if (!p4d_present(*p4d)) in page_vma_mapped_walk()
173 pud = pud_offset(p4d, pvmw->address); in page_vma_mapped_walk()
|
| D | pgtable-generic.c | 26 void p4d_clear_bad(p4d_t *p4d) in p4d_clear_bad() argument
28 p4d_ERROR(*p4d); in p4d_clear_bad()
29 p4d_clear(p4d); in p4d_clear_bad()
|
| D | userfaultfd.c | 153 p4d_t *p4d; in mm_alloc_pmd() local
157 p4d = p4d_alloc(mm, pgd, address); in mm_alloc_pmd()
158 if (!p4d) in mm_alloc_pmd()
160 pud = pud_alloc(mm, p4d, address); in mm_alloc_pmd()
|
| D | rmap.c | 713 p4d_t *p4d; in mm_find_pmd() local
722 p4d = p4d_offset(pgd, address); in mm_find_pmd()
723 if (!p4d_present(*p4d)) in mm_find_pmd()
726 pud = pud_offset(p4d, address); in mm_find_pmd()
|
| D | hugetlb.c | 4672 p4d_t *p4d = p4d_offset(pgd, *addr); in huge_pmd_unshare() local
4673 pud_t *pud = pud_offset(p4d, *addr); in huge_pmd_unshare()
4709 p4d_t *p4d; in huge_pte_alloc() local
4714 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
4715 if (!p4d) in huge_pte_alloc()
4717 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc()
4747 p4d_t *p4d; in huge_pte_offset() local
4754 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
4755 if (!p4d_present(*p4d)) in huge_pte_offset()
4758 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
| D | swapfile.c | 1879 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, in unuse_pud_range() argument
1887 pud = pud_offset(p4d, addr); in unuse_pud_range()
1903 p4d_t *p4d; in unuse_p4d_range() local
1907 p4d = p4d_offset(pgd, addr); in unuse_p4d_range()
1910 if (p4d_none_or_clear_bad(p4d)) in unuse_p4d_range()
1912 ret = unuse_pud_range(vma, p4d, addr, next, entry, page); in unuse_p4d_range()
1915 } while (p4d++, addr = next, addr != end); in unuse_p4d_range()
|
| D | huge_memory.c | 2224 p4d_t *p4d; in split_huge_pmd_address() local
2232 p4d = p4d_offset(pgd, address); in split_huge_pmd_address()
2233 if (!p4d_present(*p4d)) in split_huge_pmd_address()
2236 pud = pud_offset(p4d, address); in split_huge_pmd_address()
|