| /mm/kasan/ |
| D | init.c | 33 static inline bool kasan_p4d_table(pgd_t pgd) in kasan_p4d_table() argument
35 return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d)); in kasan_p4d_table()
38 static inline bool kasan_p4d_table(pgd_t pgd) in kasan_p4d_table() argument
179 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, in zero_p4d_populate() argument
182 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate()
231 pgd_t *pgd = pgd_offset_k(addr); in kasan_populate_early_shadow() local
250 pgd_populate(&init_mm, pgd, in kasan_populate_early_shadow()
252 p4d = p4d_offset(pgd, addr); in kasan_populate_early_shadow()
264 if (pgd_none(*pgd)) { in kasan_populate_early_shadow()
268 p = p4d_alloc(&init_mm, pgd, addr); in kasan_populate_early_shadow()
[all …]
|
| D | shadow.c | 148 pgd_t *pgd = pgd_offset_k(addr); in shadow_mapped() local
154 if (pgd_none(*pgd)) in shadow_mapped()
156 p4d = p4d_offset(pgd, addr); in shadow_mapped()
|
| /mm/ |
| D | sparse-vmemmap.c | 158 static int vmemmap_p4d_range(pgd_t *pgd, unsigned long addr, in vmemmap_p4d_range() argument
165 p4d = p4d_offset(pgd, addr); in vmemmap_p4d_range()
183 pgd_t *pgd; in vmemmap_remap_range() local
188 pgd = pgd_offset_k(addr); in vmemmap_remap_range()
193 ret = vmemmap_p4d_range(pgd, addr, next, walk); in vmemmap_remap_range()
196 } while (pgd++, addr = next, addr != end); in vmemmap_remap_range()
549 p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node) in vmemmap_p4d_populate() argument
551 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate()
563 pgd_t *pgd = pgd_offset_k(addr); in vmemmap_pgd_populate() local
564 if (pgd_none(*pgd)) { in vmemmap_pgd_populate()
[all …]
|
| D | pagewalk.c | 213 static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, in walk_p4d_range() argument
222 p4d = p4d_offset(pgd, addr); in walk_p4d_range()
251 pgd_t *pgd; in walk_pgd_range() local
256 if (walk->pgd) in walk_pgd_range()
257 pgd = walk->pgd + pgd_index(addr); in walk_pgd_range()
259 pgd = pgd_offset(walk->mm, addr); in walk_pgd_range()
262 if (pgd_none_or_clear_bad(pgd)) { in walk_pgd_range()
270 err = ops->pgd_entry(pgd, addr, next, walk); in walk_pgd_range()
274 if (is_hugepd(__hugepd(pgd_val(*pgd)))) in walk_pgd_range()
275 err = walk_hugepd_range((hugepd_t *)pgd, addr, next, walk, PGDIR_SHIFT); in walk_pgd_range()
[all …]
|
| D | pgalloc-track.h | 6 static inline p4d_t *p4d_alloc_track(struct mm_struct *mm, pgd_t *pgd, in p4d_alloc_track() argument
10 if (unlikely(pgd_none(*pgd))) { in p4d_alloc_track()
11 if (__p4d_alloc(mm, pgd, address)) in p4d_alloc_track()
16 return p4d_offset(pgd, address); in p4d_alloc_track()
|
| D | memory.c | 362 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, in free_p4d_range() argument
371 p4d = p4d_offset(pgd, addr); in free_p4d_range()
390 p4d = p4d_offset(pgd, start); in free_p4d_range()
391 pgd_clear(pgd); in free_p4d_range()
402 pgd_t *pgd; in free_pgd_range() local
451 pgd = pgd_offset(tlb->mm, addr); in free_pgd_range()
454 if (pgd_none_or_clear_bad(pgd)) in free_pgd_range()
456 free_p4d_range(tlb, pgd, addr, next, floor, ceiling); in free_pgd_range()
457 } while (pgd++, addr = next, addr != end); in free_pgd_range()
574 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); in print_bad_pte() local
[all …]
|
| D | gup.c | 840 pgd_t *pgd; in follow_page_mask() local
853 pgd = pgd_offset(mm, address); in follow_page_mask()
855 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) in follow_page_mask()
858 if (pgd_huge(*pgd)) { in follow_page_mask()
859 page = follow_huge_pgd(mm, address, pgd, flags); in follow_page_mask()
864 if (is_hugepd(__hugepd(pgd_val(*pgd)))) { in follow_page_mask()
866 __hugepd(pgd_val(*pgd)), flags, in follow_page_mask()
873 return follow_p4d_mask(vma, address, pgd, flags, ctx); in follow_page_mask()
895 pgd_t *pgd; in get_gate_page() local
906 pgd = pgd_offset_k(address); in get_gate_page()
[all …]
|
| D | ptdump.c | 28 static int ptdump_pgd_entry(pgd_t *pgd, unsigned long addr, in ptdump_pgd_entry() argument
32 pgd_t val = READ_ONCE(*pgd); in ptdump_pgd_entry()
143 void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) in ptdump_walk_pgd() argument
150 &ptdump_ops, pgd, st); in ptdump_walk_pgd()
|
| D | vmalloc.c | 260 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, in vmap_p4d_range() argument
267 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); in vmap_p4d_range()
290 pgd_t *pgd; in vmap_range_noflush() local
300 pgd = pgd_offset_k(addr); in vmap_range_noflush()
303 err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, in vmap_range_noflush()
307 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); in vmap_range_noflush()
392 static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, in vunmap_p4d_range() argument
399 p4d = p4d_offset(pgd, addr); in vunmap_p4d_range()
430 pgd_t *pgd; in vunmap_range_noflush() local
435 pgd = pgd_offset_k(addr); in vunmap_range_noflush()
[all …]
|
| D | mprotect.c | 324 struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, in change_p4d_range() argument
331 p4d = p4d_offset(pgd, addr); in change_p4d_range()
348 pgd_t *pgd; in change_protection_range() local
353 pgd = pgd_offset(mm, addr); in change_protection_range()
357 if (pgd_none_or_clear_bad(pgd)) in change_protection_range()
359 pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot, in change_protection_range()
361 } while (pgd++, addr = next, addr != end); in change_protection_range()
|
| D | debug_vm_pgtable.c | 515 pgd_t pgd; in pgd_basic_tests() local
518 memset(&pgd, RANDOM_NZVALUE, sizeof(pgd_t)); in pgd_basic_tests()
519 WARN_ON(!pgd_same(pgd, pgd)); in pgd_basic_tests()
596 pgd_t pgd = READ_ONCE(*(args->pgdp)); in pgd_clear_tests() local
602 pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE); in pgd_clear_tests()
603 WRITE_ONCE(*args->pgdp, pgd); in pgd_clear_tests()
605 pgd = READ_ONCE(*args->pgdp); in pgd_clear_tests()
606 WARN_ON(!pgd_none(pgd)); in pgd_clear_tests()
611 pgd_t pgd; in pgd_populate_tests() local
624 pgd = READ_ONCE(*args->pgdp); in pgd_populate_tests()
[all …]
|
| D | page_vma_mapped.c | 158 pgd_t *pgd; in page_vma_mapped_walk() local
197 pgd = pgd_offset(mm, pvmw->address); in page_vma_mapped_walk()
198 if (!pgd_present(*pgd)) { in page_vma_mapped_walk()
202 p4d = p4d_offset(pgd, pvmw->address); in page_vma_mapped_walk()
|
| D | pgtable-generic.c | 21 void pgd_clear_bad(pgd_t *pgd) in pgd_clear_bad() argument
23 pgd_ERROR(*pgd); in pgd_clear_bad()
24 pgd_clear(pgd); in pgd_clear_bad()
|
| D | mremap.c | 35 pgd_t *pgd; in get_old_pud() local
39 pgd = pgd_offset(mm, addr); in get_old_pud()
40 if (pgd_none_or_clear_bad(pgd)) in get_old_pud()
43 p4d = p4d_offset(pgd, addr); in get_old_pud()
73 pgd_t *pgd; in alloc_new_pud() local
76 pgd = pgd_offset(mm, addr); in alloc_new_pud()
77 p4d = p4d_alloc(mm, pgd, addr); in alloc_new_pud()
|
| D | userfaultfd.c | 264 pgd_t *pgd; in mm_alloc_pmd() local
268 pgd = pgd_offset(mm, address); in mm_alloc_pmd()
269 p4d = p4d_alloc(mm, pgd, address); in mm_alloc_pmd()
|
| D | init-mm.c | 31 .pgd = swapper_pg_dir,
|
| D | memory-failure.c | 311 pgd_t *pgd; in dev_pagemap_mapping_shift() local
317 pgd = pgd_offset(vma->vm_mm, address); in dev_pagemap_mapping_shift()
318 if (!pgd_present(*pgd)) in dev_pagemap_mapping_shift()
320 p4d = p4d_offset(pgd, address); in dev_pagemap_mapping_shift()
|
| D | hugetlb.c | 6073 pgd_t *pgd = pgd_offset(mm, *addr); in huge_pmd_unshare() local
6074 p4d_t *p4d = p4d_offset(pgd, *addr); in huge_pmd_unshare()
6124 pgd_t *pgd; in huge_pte_alloc() local
6129 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
6130 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
6162 pgd_t *pgd; in huge_pte_offset() local
6167 pgd = pgd_offset(mm, addr); in huge_pte_offset()
6168 if (!pgd_present(*pgd)) in huge_pte_offset()
6170 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
6271 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) in follow_huge_pgd() argument
[all …]
|
| D | rmap.c | 753 pgd_t *pgd; in mm_find_pmd() local
759 pgd = pgd_offset(mm, address); in mm_find_pmd()
760 if (!pgd_present(*pgd)) in mm_find_pmd()
763 p4d = p4d_offset(pgd, address); in mm_find_pmd()
|
| D | swapfile.c | 2059 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, in unuse_p4d_range() argument
2068 p4d = p4d_offset(pgd, addr); in unuse_p4d_range()
2084 pgd_t *pgd; in unuse_vma() local
2091 pgd = pgd_offset(vma->vm_mm, addr); in unuse_vma()
2094 if (pgd_none_or_clear_bad(pgd)) in unuse_vma()
2096 ret = unuse_p4d_range(vma, pgd, addr, next, type, in unuse_vma()
2100 } while (pgd++, addr = next, addr != end); in unuse_vma()
|
| D | debug.c | 241 mm->pgd, atomic_read(&mm->mm_users), in dump_mm()
|
| D | huge_memory.c | 2249 pgd_t *pgd; in split_huge_pmd_address() local
2254 pgd = pgd_offset(vma->vm_mm, address); in split_huge_pmd_address()
2255 if (!pgd_present(*pgd)) in split_huge_pmd_address()
2258 p4d = p4d_offset(pgd, address); in split_huge_pmd_address()
|