Lines Matching full:addr
96 static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr, in zero_pte_populate() argument
99 pte_t *pte = pte_offset_kernel(pmd, addr); in zero_pte_populate()
106 while (addr + PAGE_SIZE <= end) { in zero_pte_populate()
107 set_pte_at(&init_mm, addr, pte, zero_pte); in zero_pte_populate()
108 addr += PAGE_SIZE; in zero_pte_populate()
109 pte = pte_offset_kernel(pmd, addr); in zero_pte_populate()
113 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr, in zero_pmd_populate() argument
116 pmd_t *pmd = pmd_offset(pud, addr); in zero_pmd_populate()
120 next = pmd_addr_end(addr, end); in zero_pmd_populate()
122 if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { in zero_pmd_populate()
140 zero_pte_populate(pmd, addr, next); in zero_pmd_populate()
141 } while (pmd++, addr = next, addr != end); in zero_pmd_populate()
146 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument
149 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate()
153 next = pud_addr_end(addr, end); in zero_pud_populate()
154 if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { in zero_pud_populate()
159 pmd = pmd_offset(pud, addr); in zero_pud_populate()
169 p = pmd_alloc(&init_mm, pud, addr); in zero_pud_populate()
177 zero_pmd_populate(pud, addr, next); in zero_pud_populate()
178 } while (pud++, addr = next, addr != end); in zero_pud_populate()
183 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, in zero_p4d_populate() argument
186 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate()
190 next = p4d_addr_end(addr, end); in zero_p4d_populate()
191 if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { in zero_p4d_populate()
197 pud = pud_offset(p4d, addr); in zero_p4d_populate()
200 pmd = pmd_offset(pud, addr); in zero_p4d_populate()
210 p = pud_alloc(&init_mm, p4d, addr); in zero_p4d_populate()
218 zero_pud_populate(p4d, addr, next); in zero_p4d_populate()
219 } while (p4d++, addr = next, addr != end); in zero_p4d_populate()
233 unsigned long addr = (unsigned long)shadow_start; in kasan_populate_early_shadow() local
235 pgd_t *pgd = pgd_offset_k(addr); in kasan_populate_early_shadow()
239 next = pgd_addr_end(addr, end); in kasan_populate_early_shadow()
241 if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { in kasan_populate_early_shadow()
256 p4d = p4d_offset(pgd, addr); in kasan_populate_early_shadow()
259 pud = pud_offset(p4d, addr); in kasan_populate_early_shadow()
262 pmd = pmd_offset(pud, addr); in kasan_populate_early_shadow()
272 p = p4d_alloc(&init_mm, pgd, addr); in kasan_populate_early_shadow()
280 zero_p4d_populate(pgd, addr, next); in kasan_populate_early_shadow()
281 } while (pgd++, addr = next, addr != end); in kasan_populate_early_shadow()
346 static void kasan_remove_pte_table(pte_t *pte, unsigned long addr, in kasan_remove_pte_table() argument
351 for (; addr < end; addr = next, pte++) { in kasan_remove_pte_table()
352 next = (addr + PAGE_SIZE) & PAGE_MASK; in kasan_remove_pte_table()
361 pte_clear(&init_mm, addr, pte); in kasan_remove_pte_table()
365 static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr, in kasan_remove_pmd_table() argument
370 for (; addr < end; addr = next, pmd++) { in kasan_remove_pmd_table()
373 next = pmd_addr_end(addr, end); in kasan_remove_pmd_table()
379 if (IS_ALIGNED(addr, PMD_SIZE) && in kasan_remove_pmd_table()
385 pte = pte_offset_kernel(pmd, addr); in kasan_remove_pmd_table()
386 kasan_remove_pte_table(pte, addr, next); in kasan_remove_pmd_table()
391 static void kasan_remove_pud_table(pud_t *pud, unsigned long addr, in kasan_remove_pud_table() argument
396 for (; addr < end; addr = next, pud++) { in kasan_remove_pud_table()
399 next = pud_addr_end(addr, end); in kasan_remove_pud_table()
405 if (IS_ALIGNED(addr, PUD_SIZE) && in kasan_remove_pud_table()
411 pmd = pmd_offset(pud, addr); in kasan_remove_pud_table()
413 kasan_remove_pmd_table(pmd, addr, next); in kasan_remove_pud_table()
418 static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr, in kasan_remove_p4d_table() argument
423 for (; addr < end; addr = next, p4d++) { in kasan_remove_p4d_table()
426 next = p4d_addr_end(addr, end); in kasan_remove_p4d_table()
432 if (IS_ALIGNED(addr, P4D_SIZE) && in kasan_remove_p4d_table()
438 pud = pud_offset(p4d, addr); in kasan_remove_p4d_table()
439 kasan_remove_pud_table(pud, addr, next); in kasan_remove_p4d_table()
446 unsigned long addr, end, next; in kasan_remove_zero_shadow() local
449 addr = (unsigned long)kasan_mem_to_shadow(start); in kasan_remove_zero_shadow()
450 end = addr + (size >> KASAN_SHADOW_SCALE_SHIFT); in kasan_remove_zero_shadow()
457 for (; addr < end; addr = next) { in kasan_remove_zero_shadow()
460 next = pgd_addr_end(addr, end); in kasan_remove_zero_shadow()
462 pgd = pgd_offset_k(addr); in kasan_remove_zero_shadow()
467 if (IS_ALIGNED(addr, PGDIR_SIZE) && in kasan_remove_zero_shadow()
474 p4d = p4d_offset(pgd, addr); in kasan_remove_zero_shadow()
475 kasan_remove_p4d_table(p4d, addr, next); in kasan_remove_zero_shadow()