128 static bool kfence_protect(unsigned long addr)  in kfence_protect()  argument
130 	return !KFENCE_WARN_ON(!kfence_protect_page(ALIGN_DOWN(addr, PAGE_SIZE), true));  in kfence_protect()
133 static bool kfence_unprotect(unsigned long addr)  in kfence_unprotect()  argument
135 	return !KFENCE_WARN_ON(!kfence_protect_page(ALIGN_DOWN(addr, PAGE_SIZE), false));  in kfence_unprotect()
138 static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)  in addr_to_metadata()  argument
144 	if (!is_kfence_address((void *)addr))  in addr_to_metadata()
152 	index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;  in addr_to_metadata()
175 	if (KFENCE_WARN_ON(ALIGN_DOWN(meta->addr, PAGE_SIZE) != pageaddr))  in metadata_to_pageaddr()
209 static inline bool set_canary_byte(u8 *addr)  in set_canary_byte()  argument
211 	*addr = KFENCE_CANARY_PATTERN(addr);  in set_canary_byte()
216 static inline bool check_canary_byte(u8 *addr)  in check_canary_byte()  argument
218 	if (likely(*addr == KFENCE_CANARY_PATTERN(addr)))  in check_canary_byte()
222 	kfence_report_error((unsigned long)addr, false, NULL, addr_to_metadata((unsigned long)addr),  in check_canary_byte()
230 	const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE);  in for_each_canary()
231 	unsigned long addr;  in for_each_canary()  local
245 	for (addr = pageaddr; addr < meta->addr; addr++) {  in for_each_canary()
246 		if (!fn((u8 *)addr))  in for_each_canary()
251 	for (addr = meta->addr + meta->size; addr < pageaddr + PAGE_SIZE; addr++) {  in for_each_canary()
252 		if (!fn((u8 *)addr))  in for_each_canary()
262 	void *addr;  in kfence_guarded_alloc()  local
292 	meta->addr = metadata_to_pageaddr(meta);  in kfence_guarded_alloc()
295 		kfence_unprotect(meta->addr);  in kfence_guarded_alloc()
307 		meta->addr += PAGE_SIZE - size;  in kfence_guarded_alloc()
308 		meta->addr = ALIGN_DOWN(meta->addr, cache->align);  in kfence_guarded_alloc()
311 	addr = (void *)meta->addr;  in kfence_guarded_alloc()
321 	page = virt_to_page(meta->addr);  in kfence_guarded_alloc()
326 		page->s_mem = addr;  in kfence_guarded_alloc()
338 		memzero_explicit(addr, size);  in kfence_guarded_alloc()
340 		cache->ctor(addr);  in kfence_guarded_alloc()
343 		kfence_protect(meta->addr); /* Random "faults" by protecting the object. */  in kfence_guarded_alloc()
348 	return addr;  in kfence_guarded_alloc()
351 static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie)  in kfence_guarded_free()  argument
358 	if (meta->state != KFENCE_OBJECT_ALLOCATED || meta->addr != (unsigned long)addr) {  in kfence_guarded_free()
361 		kfence_report_error((unsigned long)addr, false, NULL, meta,  in kfence_guarded_free()
368 	kcsan_begin_scoped_access((void *)ALIGN_DOWN((unsigned long)addr, PAGE_SIZE), PAGE_SIZE,  in kfence_guarded_free()
373 		kfence_unprotect((unsigned long)addr); /* To check canary bytes. */  in kfence_guarded_free()
391 		memzero_explicit(addr, meta->size);  in kfence_guarded_free()
399 	kfence_protect((unsigned long)addr);  in kfence_guarded_free()
421 	kfence_guarded_free((void *)meta->addr, meta, false);  in rcu_guarded_free()
426 	unsigned long addr = (unsigned long)__kfence_pool;  in kfence_init_pool()  local
436 	pages = virt_to_page(addr);  in kfence_init_pool()
464 		if (unlikely(!kfence_protect(addr)))  in kfence_init_pool()
467 		addr += PAGE_SIZE;  in kfence_init_pool()
477 		meta->addr = addr; /* Initialize for validation in metadata_to_pageaddr(). */  in kfence_init_pool()
481 		if (unlikely(!kfence_protect(addr + PAGE_SIZE)))  in kfence_init_pool()
484 		addr += 2 * PAGE_SIZE;  in kfence_init_pool()
505 	memblock_free_late(__pa(addr), KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool));  in kfence_init_pool()
717 			kfence_guarded_free((void *)meta->addr, meta, /*zombie=*/true);  in kfence_shutdown_cache()
780 size_t kfence_ksize(const void *addr)  in kfence_ksize()  argument
782 	const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in kfence_ksize()
791 void *kfence_object_start(const void *addr)  in kfence_object_start()  argument
793 	const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in kfence_object_start()
799 	return meta ? (void *)meta->addr : NULL;  in kfence_object_start()
802 void __kfence_free(void *addr)  in __kfence_free()  argument
804 	struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in __kfence_free()
815 		kfence_guarded_free(addr, meta, false);  in __kfence_free()
818 bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs)  in kfence_handle_page_fault()  argument
820 	const int page_index = (addr - (unsigned long)__kfence_pool) / PAGE_SIZE;  in kfence_handle_page_fault()
825 	if (!is_kfence_address((void *)addr))  in kfence_handle_page_fault()
829 		return kfence_unprotect(addr); /* ... unprotect and proceed. */  in kfence_handle_page_fault()
838 		meta = addr_to_metadata(addr - PAGE_SIZE);  in kfence_handle_page_fault()
842 			distance = addr - data_race(meta->addr + meta->size);  in kfence_handle_page_fault()
845 		meta = addr_to_metadata(addr + PAGE_SIZE);  in kfence_handle_page_fault()
848 			if (!to_report || distance > data_race(meta->addr) - addr)  in kfence_handle_page_fault()
856 		to_report->unprotected_page = addr;  in kfence_handle_page_fault()
865 		to_report = addr_to_metadata(addr);  in kfence_handle_page_fault()
881 		kfence_report_error(addr, is_write, regs, to_report, error_type);  in kfence_handle_page_fault()
885 		kfence_report_error(addr, is_write, regs, NULL, KFENCE_ERROR_INVALID);  in kfence_handle_page_fault()
888 	return kfence_unprotect(addr); /* Unprotect and let access proceed. */  in kfence_handle_page_fault()