Lines Matching refs:object
338 static inline void *get_freepointer(struct kmem_cache *s, void *object) in get_freepointer() argument
340 object = kasan_reset_tag(object); in get_freepointer()
341 return freelist_dereference(s, object + s->offset); in get_freepointer()
344 static void prefetch_freepointer(const struct kmem_cache *s, void *object) in prefetch_freepointer() argument
346 prefetch(object + s->offset); in prefetch_freepointer()
349 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) in get_freepointer_safe() argument
355 return get_freepointer(s, object); in get_freepointer_safe()
357 object = kasan_reset_tag(object); in get_freepointer_safe()
358 freepointer_addr = (unsigned long)object + s->offset; in get_freepointer_safe()
363 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) in set_freepointer() argument
365 unsigned long freeptr_addr = (unsigned long)object + s->offset; in set_freepointer()
368 BUG_ON(object == fp); /* naive detection of double free or corruption */ in set_freepointer()
628 struct page *page, void *object) in check_valid_pointer() argument
632 if (!object) in check_valid_pointer()
636 object = kasan_reset_tag(object); in check_valid_pointer()
637 object = restore_red_left(s, object); in check_valid_pointer()
638 if (object < base || object >= base + page->objects * s->size || in check_valid_pointer()
639 (object - base) % s->size) { in check_valid_pointer()
675 static struct track *get_track(struct kmem_cache *s, void *object, in get_track() argument
680 p = object + get_info_end(s); in get_track()
721 static void set_track(struct kmem_cache *s, void *object, in set_track() argument
724 struct track *p = get_track(s, object, alloc); in set_track()
747 static void init_tracking(struct kmem_cache *s, void *object) in init_tracking() argument
752 set_track(s, object, TRACK_FREE, 0UL); in init_tracking()
753 set_track(s, object, TRACK_ALLOC, 0UL); in init_tracking()
775 void print_tracking(struct kmem_cache *s, void *object) in print_tracking() argument
781 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time); in print_tracking()
782 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time); in print_tracking()
877 u8 *object, char *reason) in object_err() argument
883 print_trailer(s, page, object); in object_err()
905 static void init_object(struct kmem_cache *s, void *object, u8 val) in init_object() argument
907 u8 *p = kasan_reset_tag(object); in init_object()
929 u8 *object, char *what, in check_bytes_and_report() argument
953 print_trailer(s, page, object); in check_bytes_and_report()
1054 void *object, u8 val) in check_object() argument
1056 u8 *p = object; in check_object()
1057 u8 *endobject = object + s->object_size; in check_object()
1060 if (!check_bytes_and_report(s, page, object, "Left Redzone", in check_object()
1061 object - s->red_left_pad, val, s->red_left_pad)) in check_object()
1064 if (!check_bytes_and_report(s, page, object, "Right Redzone", in check_object()
1142 void *object = NULL; in on_freelist() local
1150 if (object) { in on_freelist()
1151 object_err(s, page, object, in on_freelist()
1153 set_freepointer(s, object, NULL); in on_freelist()
1163 object = fp; in on_freelist()
1164 fp = get_freepointer(s, object); in on_freelist()
1187 static void trace(struct kmem_cache *s, struct page *page, void *object, in trace() argument
1194 object, page->inuse, in trace()
1198 print_section(KERN_INFO, "Object ", (void *)object, in trace()
1265 void *object) in setup_object_debug() argument
1270 init_object(s, object, SLUB_RED_INACTIVE); in setup_object_debug()
1271 init_tracking(s, object); in setup_object_debug()
1286 struct page *page, void *object) in alloc_consistency_checks() argument
1291 if (!check_valid_pointer(s, page, object)) { in alloc_consistency_checks()
1292 object_err(s, page, object, "Freelist Pointer check fails"); in alloc_consistency_checks()
1296 if (!check_object(s, page, object, SLUB_RED_INACTIVE)) in alloc_consistency_checks()
1304 void *object, unsigned long addr) in alloc_debug_processing() argument
1307 if (!alloc_consistency_checks(s, page, object)) in alloc_debug_processing()
1313 set_track(s, object, TRACK_ALLOC, addr); in alloc_debug_processing()
1314 trace(s, page, object, 1); in alloc_debug_processing()
1315 init_object(s, object, SLUB_RED_ACTIVE); in alloc_debug_processing()
1333 struct page *page, void *object, unsigned long addr) in free_consistency_checks() argument
1335 if (!check_valid_pointer(s, page, object)) { in free_consistency_checks()
1336 slab_err(s, page, "Invalid object pointer 0x%p", object); in free_consistency_checks()
1340 if (on_freelist(s, page, object)) { in free_consistency_checks()
1341 object_err(s, page, object, "Object already free"); in free_consistency_checks()
1345 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) in free_consistency_checks()
1351 object); in free_consistency_checks()
1354 object); in free_consistency_checks()
1357 object_err(s, page, object, in free_consistency_checks()
1371 void *object = head; in free_debug_processing() local
1388 if (!free_consistency_checks(s, page, object, addr)) in free_debug_processing()
1393 set_track(s, object, TRACK_FREE, addr); in free_debug_processing()
1394 trace(s, page, object, 0); in free_debug_processing()
1396 init_object(s, object, SLUB_RED_INACTIVE); in free_debug_processing()
1399 if (object != tail) { in free_debug_processing()
1400 object = get_freepointer(s, object); in free_debug_processing()
1413 slab_fix(s, "Object at 0x%p not freed", object); in free_debug_processing()
1625 struct page *page, void *object) {} in setup_object_debug() argument
1630 struct page *page, void *object, unsigned long addr) { return 0; } in alloc_debug_processing() argument
1640 void *object, u8 val) { return 1; } in check_object() argument
1729 void *object; in slab_free_freelist_hook() local
1743 object = next; in slab_free_freelist_hook()
1744 next = get_freepointer(s, object); in slab_free_freelist_hook()
1747 if (!slab_free_hook(s, object, slab_want_init_on_free(s))) { in slab_free_freelist_hook()
1749 set_freepointer(s, object, *head); in slab_free_freelist_hook()
1750 *head = object; in slab_free_freelist_hook()
1752 *tail = object; in slab_free_freelist_hook()
1760 } while (object != old_tail); in slab_free_freelist_hook()
1769 void *object) in setup_object() argument
1771 setup_object_debug(s, page, object); in setup_object()
1772 object = kasan_init_slab_obj(s, object); in setup_object()
1774 kasan_unpoison_object_data(s, object); in setup_object()
1775 s->ctor(object); in setup_object()
1776 kasan_poison_object_data(s, object); in setup_object()
1778 return object; in setup_object()
2129 void *object = NULL; in get_partial_node() local
2150 t = acquire_slab(s, n, page, object == NULL, &objects); in get_partial_node()
2155 if (!object) { in get_partial_node()
2158 object = t; in get_partial_node()
2169 return object; in get_partial_node()
2183 void *object; in get_any_partial() local
2218 object = get_partial_node(s, n, ret_page, flags); in get_any_partial()
2219 if (object) { in get_any_partial()
2227 return object; in get_any_partial()
2242 void *object; in get_partial() local
2248 object = get_partial_node(s, get_node(s, searchnode), ret_page, flags); in get_partial()
2249 if (object || node != NUMA_NO_NODE) in get_partial()
2250 return object; in get_partial()
3143 void *object; in slab_alloc_node() local
3154 object = kfence_alloc(s, orig_size, gfpflags); in slab_alloc_node()
3155 if (unlikely(object)) in slab_alloc_node()
3191 object = c->freelist; in slab_alloc_node()
3201 unlikely(!object || !page || !node_match(page, node))) { in slab_alloc_node()
3202 object = __slab_alloc(s, gfpflags, node, addr, c); in slab_alloc_node()
3204 void *next_object = get_freepointer_safe(s, object); in slab_alloc_node()
3222 object, tid, in slab_alloc_node()
3232 maybe_wipe_obj_freeptr(s, object); in slab_alloc_node()
3236 slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init); in slab_alloc_node()
3238 return object; in slab_alloc_node()
3544 static inline void free_nonslab_page(struct page *page, void *object) in free_nonslab_page() argument
3549 kfree_hook(object); in free_nonslab_page()
3572 void *object; in build_detached_freelist() local
3579 object = p[--size]; in build_detached_freelist()
3581 } while (!object && size); in build_detached_freelist()
3583 if (!object) in build_detached_freelist()
3586 page = virt_to_head_page(object); in build_detached_freelist()
3590 free_nonslab_page(page, object); in build_detached_freelist()
3597 df->s = cache_from_obj(s, object); /* Support for memcg */ in build_detached_freelist()
3600 if (is_kfence_address(object)) { in build_detached_freelist()
3601 slab_free_hook(df->s, object, false); in build_detached_freelist()
3602 __kfence_free(object); in build_detached_freelist()
3609 set_freepointer(df->s, object, NULL); in build_detached_freelist()
3610 df->tail = object; in build_detached_freelist()
3611 df->freelist = object; in build_detached_freelist()
3616 object = p[--size]; in build_detached_freelist()
3617 if (!object) in build_detached_freelist()
3621 if (df->page == virt_to_head_page(object)) { in build_detached_freelist()
3623 set_freepointer(df->s, object, df->freelist); in build_detached_freelist()
3624 df->freelist = object; in build_detached_freelist()
3682 void *object = kfence_alloc(s, s->object_size, flags); in kmem_cache_alloc_bulk() local
3684 if (unlikely(object)) { in kmem_cache_alloc_bulk()
3685 p[i] = object; in kmem_cache_alloc_bulk()
3689 object = c->freelist; in kmem_cache_alloc_bulk()
3690 if (unlikely(!object)) { in kmem_cache_alloc_bulk()
3718 c->freelist = get_freepointer(s, object); in kmem_cache_alloc_bulk()
3719 p[i] = object; in kmem_cache_alloc_bulk()
4325 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page) in __kmem_obj_info() argument
4335 kpp->kp_ptr = object; in __kmem_obj_info()
4339 objp0 = kasan_reset_tag(object); in __kmem_obj_info()
4542 size_t __ksize(const void *object) in __ksize() argument
4546 if (unlikely(object == ZERO_SIZE_PTR)) in __ksize()
4549 page = virt_to_head_page(object); in __ksize()
4563 void *object = (void *)x; in kfree() local
4572 free_nonslab_page(page, object); in kfree()
4575 slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_); in kfree()