62 static inline bool kasan_pmd_table(pud_t pud)  in kasan_pmd_table()  argument
64 	return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd));  in kasan_pmd_table()
67 static inline bool kasan_pmd_table(pud_t pud)  in kasan_pmd_table()  argument
113 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,  in zero_pmd_populate()  argument
116 	pmd_t *pmd = pmd_offset(pud, addr);  in zero_pmd_populate()
149 	pud_t *pud = pud_offset(p4d, addr);  in zero_pud_populate()  local
157 			pud_populate(&init_mm, pud,  in zero_pud_populate()
159 			pmd = pmd_offset(pud, addr);  in zero_pud_populate()
165 		if (pud_none(*pud)) {  in zero_pud_populate()
169 				p = pmd_alloc(&init_mm, pud, addr);  in zero_pud_populate()
173 				pud_populate(&init_mm, pud,  in zero_pud_populate()
177 		zero_pmd_populate(pud, addr, next);  in zero_pud_populate()
178 	} while (pud++, addr = next, addr != end);  in zero_pud_populate()
192 			pud_t *pud;  in zero_p4d_populate()  local
197 			pud = pud_offset(p4d, addr);  in zero_p4d_populate()
198 			pud_populate(&init_mm, pud,  in zero_p4d_populate()
200 			pmd = pmd_offset(pud, addr);  in zero_p4d_populate()
243 			pud_t *pud;  in kasan_populate_early_shadow()  local
259 			pud = pud_offset(p4d, addr);  in kasan_populate_early_shadow()
260 			pud_populate(&init_mm, pud,  in kasan_populate_early_shadow()
262 			pmd = pmd_offset(pud, addr);  in kasan_populate_early_shadow()
301 static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud)  in kasan_free_pmd()  argument
312 	pmd_free(&init_mm, (pmd_t *)page_to_virt(pud_page(*pud)));  in kasan_free_pmd()
313 	pud_clear(pud);  in kasan_free_pmd()
318 	pud_t *pud;  in kasan_free_pud()  local
322 		pud = pud_start + i;  in kasan_free_pud()
323 		if (!pud_none(*pud))  in kasan_free_pud()
391 static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,  in kasan_remove_pud_table()  argument
396 	for (; addr < end; addr = next, pud++) {  in kasan_remove_pud_table()
401 		if (!pud_present(*pud))  in kasan_remove_pud_table()
404 		if (kasan_pmd_table(*pud)) {  in kasan_remove_pud_table()
407 				pud_clear(pud);  in kasan_remove_pud_table()
411 		pmd = pmd_offset(pud, addr);  in kasan_remove_pud_table()
412 		pmd_base = pmd_offset(pud, 0);  in kasan_remove_pud_table()
414 		kasan_free_pmd(pmd_base, pud);  in kasan_remove_pud_table()
424 		pud_t *pud;  in kasan_remove_p4d_table()  local
438 		pud = pud_offset(p4d, addr);  in kasan_remove_p4d_table()
439 		kasan_remove_pud_table(pud, addr, next);  in kasan_remove_p4d_table()