| /arch/mn10300/mm/ |
| D | pgtable.c | 37 pgd_t *pgd; in set_pmd_pfn() local
49 pgd = swapper_pg_dir + pgd_index(vaddr); in set_pmd_pfn()
50 if (pgd_none(*pgd)) { in set_pmd_pfn()
54 pud = pud_offset(pgd, vaddr); in set_pmd_pfn()
108 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() argument
110 struct page *page = virt_to_page(pgd); in pgd_list_add()
118 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() argument
120 struct page *next, **pprev, *page = virt_to_page(pgd); in pgd_list_del()
128 void pgd_ctor(void *pgd) in pgd_ctor() argument
135 memcpy((pgd_t *)pgd + USER_PTRS_PER_PGD, in pgd_ctor()
[all …]
|
| /arch/powerpc/include/asm/nohash/64/ |
| D | pgtable-4k.h | 56 #define pgd_none(pgd) (!pgd_val(pgd)) argument
57 #define pgd_bad(pgd) (pgd_val(pgd) == 0) argument
58 #define pgd_present(pgd) (pgd_val(pgd) != 0) argument
59 #define pgd_page_vaddr(pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) argument
68 static inline pte_t pgd_pte(pgd_t pgd) in pgd_pte() argument
70 return __pte(pgd_val(pgd)); in pgd_pte()
77 extern struct page *pgd_page(pgd_t pgd);
|
| /arch/arm/mm/ |
| D | pgd.c | 24 #define __pgd_free(pgd) kfree(pgd) argument
27 #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) argument
121 pgd_t *pgd; in pgd_free() local
129 pgd = pgd_base + pgd_index(0); in pgd_free()
130 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
133 pud = pud_offset(pgd, 0); in pgd_free()
150 pgd_clear(pgd); in pgd_free()
157 for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { in pgd_free()
158 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
160 if (pgd_val(*pgd) & L_PGD_SWAPPER) in pgd_free()
[all …]
|
| /arch/arm/include/asm/ |
| D | stage2_pgtable.h | 22 #define stage2_pgd_none(pgd) pgd_none(pgd) argument
23 #define stage2_pgd_clear(pgd) pgd_clear(pgd) argument
24 #define stage2_pgd_present(pgd) pgd_present(pgd) argument
25 #define stage2_pgd_populate(pgd, pud) pgd_populate(NULL, pgd, pud) argument
26 #define stage2_pud_offset(pgd, address) pud_offset(pgd, address) argument
|
| /arch/frv/mm/ |
| D | pgalloc.c | 89 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() argument
91 struct page *page = virt_to_page(pgd); in pgd_list_add()
99 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() argument
101 struct page *next, **pprev, *page = virt_to_page(pgd); in pgd_list_del()
109 void pgd_ctor(void *pgd) in pgd_ctor() argument
116 memcpy((pgd_t *) pgd + USER_PGDS_IN_LAST_PML4, in pgd_ctor()
123 pgd_list_add(pgd); in pgd_ctor()
125 memset(pgd, 0, USER_PGDS_IN_LAST_PML4 * sizeof(pgd_t)); in pgd_ctor()
129 void pgd_dtor(void *pgd) in pgd_dtor() argument
134 pgd_list_del(pgd); in pgd_dtor()
[all …]
|
| /arch/sh/mm/ |
| D | hugetlbpage.c | 28 pgd_t *pgd; in huge_pte_alloc() local
33 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
34 if (pgd) { in huge_pte_alloc()
35 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
49 pgd_t *pgd; in huge_pte_offset() local
54 pgd = pgd_offset(mm, addr); in huge_pte_offset()
55 if (pgd) { in huge_pte_offset()
56 pud = pud_offset(pgd, addr); in huge_pte_offset()
|
| /arch/x86/mm/ |
| D | pti.c | 108 pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) in __pti_set_user_pgd() argument
120 return pgd; in __pti_set_user_pgd()
126 kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; in __pti_set_user_pgd()
141 if ((pgd.pgd & (_PAGE_USER|_PAGE_PRESENT)) == (_PAGE_USER|_PAGE_PRESENT) && in __pti_set_user_pgd()
143 pgd.pgd |= _PAGE_NX; in __pti_set_user_pgd()
146 return pgd; in __pti_set_user_pgd()
157 pgd_t *pgd = kernel_to_user_pgdp(pgd_offset_k(address)); in pti_user_pagetable_walk_p4d() local
165 if (pgd_none(*pgd)) { in pti_user_pagetable_walk_p4d()
170 set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); in pti_user_pagetable_walk_p4d()
172 BUILD_BUG_ON(pgd_large(*pgd) != 0); in pti_user_pagetable_walk_p4d()
[all …]
|
| D | pgtable.c | 97 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() argument
99 struct page *page = virt_to_page(pgd); in pgd_list_add()
104 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() argument
106 struct page *page = virt_to_page(pgd); in pgd_list_del()
115 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) in pgd_set_mm() argument
117 BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm)); in pgd_set_mm()
118 virt_to_page(pgd)->index = (pgoff_t)mm; in pgd_set_mm()
126 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) in pgd_ctor() argument
134 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, in pgd_ctor()
141 pgd_set_mm(pgd, mm); in pgd_ctor()
[all …]
|
| D | kasan_init_64.c | 121 static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, in kasan_populate_pgd() argument
128 if (pgd_none(*pgd)) { in kasan_populate_pgd()
130 pgd_populate(&init_mm, pgd, p); in kasan_populate_pgd()
133 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd()
143 pgd_t *pgd; in kasan_populate_shadow() local
148 pgd = pgd_offset_k(addr); in kasan_populate_shadow()
151 kasan_populate_pgd(pgd, addr, next, nid); in kasan_populate_shadow()
152 } while (pgd++, addr = next, addr != end); in kasan_populate_shadow()
169 pgd_t *pgd; in clear_pgds() local
174 pgd = pgd_offset_k(start); in clear_pgds()
[all …]
|
| /arch/metag/include/asm/ |
| D | mmu_context.h | 28 mm->context.pgd_base = (unsigned long) mm->pgd; in init_new_context()
56 static inline void load_pgd(pgd_t *pgd, int thread) in load_pgd() argument
70 metag_out32(__pa(pgd) & MMCU_TBLPHYS1_ADDR_BITS, phys1); in load_pgd()
77 load_pgd(next->pgd, hard_processor_id()); in switch_mmu()
83 if (prev->context.pgd_base != (unsigned long) prev->pgd) { in switch_mmu()
85 ((pgd_t *) prev->context.pgd_base)[i] = prev->pgd[i]; in switch_mmu()
87 prev->pgd = (pgd_t *)mmu_get_base(); in switch_mmu()
89 next->pgd = prev->pgd; in switch_mmu()
90 prev->pgd = (pgd_t *) prev->context.pgd_base; in switch_mmu()
93 next->pgd[i] = ((pgd_t *) next->context.pgd_base)[i]; in switch_mmu()
|
| D | pgalloc.h | 20 static inline void pgd_ctor(pgd_t *pgd) in pgd_ctor() argument
22 memcpy(pgd + USER_PTRS_PER_PGD, in pgd_ctor()
32 pgd_t *pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL); in pgd_alloc() local
33 if (pgd) in pgd_alloc()
34 pgd_ctor(pgd); in pgd_alloc()
35 return pgd; in pgd_alloc()
38 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
40 free_page((unsigned long)pgd); in pgd_free()
|
| /arch/arm64/include/asm/ |
| D | stage2_pgtable-nopud.h | 27 #define stage2_pgd_none(pgd) (0) argument
28 #define stage2_pgd_present(pgd) (1) argument
29 #define stage2_pgd_clear(pgd) do { } while (0) argument
30 #define stage2_pgd_populate(pgd, pud) do { } while (0) argument
32 #define stage2_pud_offset(pgd, address) ((pud_t *)(pgd)) argument
|
| D | stage2_pgtable.h | 80 #define stage2_pgd_none(pgd) pgd_none(pgd) argument
81 #define stage2_pgd_clear(pgd) pgd_clear(pgd) argument
82 #define stage2_pgd_present(pgd) pgd_present(pgd) argument
83 #define stage2_pgd_populate(pgd, pud) pgd_populate(NULL, pgd, pud) argument
84 #define stage2_pud_offset(pgd, address) pud_offset(pgd, address) argument
|
| /arch/parisc/include/asm/ |
| D | pgalloc.h | 24 pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, in pgd_alloc() local
26 pgd_t *actual_pgd = pgd; in pgd_alloc()
28 if (likely(pgd != NULL)) { in pgd_alloc()
29 memset(pgd, 0, PAGE_SIZE<<PGD_ALLOC_ORDER); in pgd_alloc()
38 + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT)); in pgd_alloc()
41 __pgd_val_set(*pgd, PxD_FLAG_ATTACHED); in pgd_alloc()
47 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
50 pgd -= PTRS_PER_PGD; in pgd_free()
52 free_pages((unsigned long)pgd, PGD_ALLOC_ORDER); in pgd_free()
59 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) in pgd_populate() argument
[all …]
|
| /arch/arm64/mm/ |
| D | mmu.c | 267 static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, in alloc_init_pud() argument
275 if (pgd_none(*pgd)) { in alloc_init_pud()
279 __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); in alloc_init_pud()
281 BUG_ON(pgd_bad(*pgd)); in alloc_init_pud()
283 pud = pud_set_fixmap_offset(pgd, addr); in alloc_init_pud()
322 pgd_t *pgd = pgd_offset_raw(pgdir, virt); in __create_pgd_mapping() local
338 alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, in __create_pgd_mapping()
341 } while (pgd++, addr = next, addr != end); in __create_pgd_mapping()
368 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, in create_mapping_noalloc()
383 __create_pgd_mapping(mm->pgd, phys, virt, size, prot, in create_pgd_mapping()
[all …]
|
| D | kasan_init.c | 72 static void __init kasan_early_pud_populate(pgd_t *pgd, in kasan_early_pud_populate() argument
79 if (pgd_none(*pgd)) in kasan_early_pud_populate()
80 __pgd_populate(pgd, __pa_symbol(kasan_zero_pud), PUD_TYPE_TABLE); in kasan_early_pud_populate()
82 pud = pud_offset_kimg(pgd, addr); in kasan_early_pud_populate()
94 pgd_t *pgd; in kasan_map_early_shadow() local
96 pgd = pgd_offset_k(addr); in kasan_map_early_shadow()
99 kasan_early_pud_populate(pgd, addr, next); in kasan_map_early_shadow()
100 } while (pgd++, addr = next, addr != end); in kasan_map_early_shadow()
116 pgd_t *pgd, *pgd_new, *pgd_end; in kasan_copy_shadow() local
118 pgd = pgd_offset_k(KASAN_SHADOW_START); in kasan_copy_shadow()
[all …]
|
| /arch/mips/mm/ |
| D | hugetlbpage.c | 27 pgd_t *pgd; in huge_pte_alloc() local
31 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
32 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
42 pgd_t *pgd; in huge_pte_offset() local
46 pgd = pgd_offset(mm, addr); in huge_pte_offset()
47 if (pgd_present(*pgd)) { in huge_pte_offset()
48 pud = pud_offset(pgd, addr); in huge_pte_offset()
|
| /arch/hexagon/include/asm/ |
| D | pgalloc.h | 36 pgd_t *pgd; in pgd_alloc() local
38 pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); in pgd_alloc()
48 memcpy(pgd, swapper_pg_dir, PTRS_PER_PGD*sizeof(pgd_t)); in pgd_alloc()
52 mm->context.ptbase = __pa(pgd); in pgd_alloc()
54 return pgd; in pgd_alloc()
57 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
59 free_page((unsigned long) pgd); in pgd_free()
135 pmdindex = (pgd_t *)pmd - mm->pgd; in pmd_populate_kernel()
136 ppmd = (pmd_t *)current->active_mm->pgd + pmdindex; in pmd_populate_kernel()
|
| /arch/um/kernel/ |
| D | mem.c | 90 pgd_t *pgd; in fixrange_init() local
99 pgd = pgd_base + i; in fixrange_init()
101 for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) { in fixrange_init()
102 pud = pud_offset(pgd, vaddr); in fixrange_init()
118 pgd_t *pgd; in fixaddr_user_init() local
134 pgd = swapper_pg_dir + pgd_index(vaddr); in fixaddr_user_init()
135 pud = pud_offset(pgd, vaddr); in fixaddr_user_init()
187 pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL); in pgd_alloc() local
189 if (pgd) { in pgd_alloc()
190 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_alloc()
[all …]
|
| /arch/tile/mm/ |
| D | pgtable.c | 55 pgd_t *pgd; in shatter_huge_page() local
67 pgd = swapper_pg_dir + pgd_index(addr); in shatter_huge_page()
68 pud = pud_offset(pgd, addr); in shatter_huge_page()
90 pgd = list_to_pgd(pos) + pgd_index(addr); in shatter_huge_page()
91 pud = pud_offset(pgd, addr); in shatter_huge_page()
123 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() argument
125 list_add(pgd_to_list(pgd), &pgd_list); in pgd_list_add()
128 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() argument
130 list_del(pgd_to_list(pgd)); in pgd_list_del()
136 static void pgd_ctor(pgd_t *pgd) in pgd_ctor() argument
[all …]
|
| /arch/x86/include/asm/ |
| D | pgtable_64.h | 200 pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd);
207 static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgd() argument
210 return pgd; in pti_set_user_pgd()
211 return __pti_set_user_pgd(pgdp, pgd); in pti_set_user_pgd()
214 static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgd() argument
216 return pgd; in pti_set_user_pgd()
223 WRITE_ONCE(p4dp->pgd, pti_set_user_pgd(&p4dp->pgd, p4d.pgd)); in native_set_p4d()
234 native_set_p4d(p4d, (p4d_t) { .pgd = native_make_pgd(0)}); in native_p4d_clear()
238 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument
241 WRITE_ONCE(*pgdp, pti_set_user_pgd(pgdp, pgd)); in native_set_pgd()
[all …]
|
| /arch/powerpc/include/asm/book3s/64/ |
| D | pgalloc.h | 65 static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) in radix__pgd_free() argument
68 free_page((unsigned long)pgd); in radix__pgd_free()
70 free_pages((unsigned long)pgd, 4); in radix__pgd_free()
76 pgd_t *pgd; in pgd_alloc() local
81 pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc()
83 memset(pgd, 0, PGD_TABLE_SIZE); in pgd_alloc()
85 return pgd; in pgd_alloc()
88 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
91 return radix__pgd_free(mm, pgd); in pgd_free()
92 kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); in pgd_free()
[all …]
|
| /arch/mips/include/asm/ |
| D | pgtable-64.h | 197 static inline int pgd_none(pgd_t pgd) in pgd_none() argument
199 return pgd_val(pgd) == (unsigned long)invalid_pud_table; in pgd_none()
202 static inline int pgd_bad(pgd_t pgd) in pgd_bad() argument
204 if (unlikely(pgd_val(pgd) & ~PAGE_MASK)) in pgd_bad()
210 static inline int pgd_present(pgd_t pgd) in pgd_present() argument
212 return pgd_val(pgd) != (unsigned long)invalid_pud_table; in pgd_present()
222 static inline unsigned long pgd_page_vaddr(pgd_t pgd) in pgd_page_vaddr() argument
224 return pgd_val(pgd); in pgd_page_vaddr()
227 #define pgd_phys(pgd) virt_to_phys((void *)pgd_val(pgd)) argument
228 #define pgd_page(pgd) (pfn_to_page(pgd_phys(pgd) >> PAGE_SHIFT)) argument
[all …]
|
| /arch/m32r/include/asm/ |
| D | pgtable-2level.h | 35 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument
36 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument
37 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument
55 #define pgd_page_vaddr(pgd) \ argument
56 ((unsigned long) __va(pgd_val(pgd) & PAGE_MASK))
59 #define pgd_page(pgd) (mem_map + ((pgd_val(pgd) >> PAGE_SHIFT) - PFN_BASE)) argument
|
| /arch/x86/kernel/ |
| D | machine_kexec_32.c | 59 free_page((unsigned long)image->arch.pgd); in machine_kexec_free_page_tables()
60 image->arch.pgd = NULL; in machine_kexec_free_page_tables()
75 image->arch.pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL); in machine_kexec_alloc_page_tables()
82 if (!image->arch.pgd || in machine_kexec_alloc_page_tables()
93 pgd_t *pgd, pmd_t *pmd, pte_t *pte, in machine_kexec_page_table_set_one() argument
99 pgd += pgd_index(vaddr); in machine_kexec_page_table_set_one()
101 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) in machine_kexec_page_table_set_one()
102 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT)); in machine_kexec_page_table_set_one()
104 p4d = p4d_offset(pgd, vaddr); in machine_kexec_page_table_set_one()
123 image->arch.pgd, pmd, image->arch.pte0, in machine_kexec_prepare_page_tables()
[all …]
|