Lines Matching refs:object
278 static inline void *get_freepointer(struct kmem_cache *s, void *object) in get_freepointer() argument
280 return freelist_dereference(s, object + s->offset); in get_freepointer()
283 static void prefetch_freepointer(const struct kmem_cache *s, void *object) in prefetch_freepointer() argument
285 prefetch(object + s->offset); in prefetch_freepointer()
288 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) in get_freepointer_safe() argument
294 return get_freepointer(s, object); in get_freepointer_safe()
296 freepointer_addr = (unsigned long)object + s->offset; in get_freepointer_safe()
301 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) in set_freepointer() argument
303 unsigned long freeptr_addr = (unsigned long)object + s->offset; in set_freepointer()
306 BUG_ON(object == fp); /* naive detection of double free or corruption */ in set_freepointer()
509 struct page *page, void *object) in check_valid_pointer() argument
513 if (!object) in check_valid_pointer()
517 object = kasan_reset_tag(object); in check_valid_pointer()
518 object = restore_red_left(s, object); in check_valid_pointer()
519 if (object < base || object >= base + page->objects * s->size || in check_valid_pointer()
520 (object - base) % s->size) { in check_valid_pointer()
536 static struct track *get_track(struct kmem_cache *s, void *object, in get_track() argument
542 p = object + s->offset + sizeof(void *); in get_track()
544 p = object + s->inuse; in get_track()
549 static void set_track(struct kmem_cache *s, void *object, in set_track() argument
552 struct track *p = get_track(s, object, alloc); in set_track()
574 static void init_tracking(struct kmem_cache *s, void *object) in init_tracking() argument
579 set_track(s, object, TRACK_FREE, 0UL); in init_tracking()
580 set_track(s, object, TRACK_ALLOC, 0UL); in init_tracking()
602 static void print_tracking(struct kmem_cache *s, void *object) in print_tracking() argument
608 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time); in print_tracking()
609 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time); in print_tracking()
690 u8 *object, char *reason) in object_err() argument
693 print_trailer(s, page, object); in object_err()
710 static void init_object(struct kmem_cache *s, void *object, u8 val) in init_object() argument
712 u8 *p = object; in init_object()
734 u8 *object, char *what, in check_bytes_and_report() argument
753 print_trailer(s, page, object); in check_bytes_and_report()
855 void *object, u8 val) in check_object() argument
857 u8 *p = object; in check_object()
858 u8 *endobject = object + s->object_size; in check_object()
861 if (!check_bytes_and_report(s, page, object, "Redzone", in check_object()
862 object - s->red_left_pad, val, s->red_left_pad)) in check_object()
865 if (!check_bytes_and_report(s, page, object, "Redzone", in check_object()
945 void *object = NULL; in on_freelist() local
953 if (object) { in on_freelist()
954 object_err(s, page, object, in on_freelist()
956 set_freepointer(s, object, NULL); in on_freelist()
966 object = fp; in on_freelist()
967 fp = get_freepointer(s, object); in on_freelist()
990 static void trace(struct kmem_cache *s, struct page *page, void *object, in trace() argument
997 object, page->inuse, in trace()
1001 print_section(KERN_INFO, "Object ", (void *)object, in trace()
1068 void *object) in setup_object_debug() argument
1073 init_object(s, object, SLUB_RED_INACTIVE); in setup_object_debug()
1074 init_tracking(s, object); in setup_object_debug()
1089 struct page *page, void *object) in alloc_consistency_checks() argument
1094 if (!check_valid_pointer(s, page, object)) { in alloc_consistency_checks()
1095 object_err(s, page, object, "Freelist Pointer check fails"); in alloc_consistency_checks()
1099 if (!check_object(s, page, object, SLUB_RED_INACTIVE)) in alloc_consistency_checks()
1107 void *object, unsigned long addr) in alloc_debug_processing() argument
1110 if (!alloc_consistency_checks(s, page, object)) in alloc_debug_processing()
1116 set_track(s, object, TRACK_ALLOC, addr); in alloc_debug_processing()
1117 trace(s, page, object, 1); in alloc_debug_processing()
1118 init_object(s, object, SLUB_RED_ACTIVE); in alloc_debug_processing()
1136 struct page *page, void *object, unsigned long addr) in free_consistency_checks() argument
1138 if (!check_valid_pointer(s, page, object)) { in free_consistency_checks()
1139 slab_err(s, page, "Invalid object pointer 0x%p", object); in free_consistency_checks()
1143 if (on_freelist(s, page, object)) { in free_consistency_checks()
1144 object_err(s, page, object, "Object already free"); in free_consistency_checks()
1148 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) in free_consistency_checks()
1154 object); in free_consistency_checks()
1157 object); in free_consistency_checks()
1160 object_err(s, page, object, in free_consistency_checks()
1174 void *object = head; in free_debug_processing() local
1191 if (!free_consistency_checks(s, page, object, addr)) in free_debug_processing()
1196 set_track(s, object, TRACK_FREE, addr); in free_debug_processing()
1197 trace(s, page, object, 0); in free_debug_processing()
1199 init_object(s, object, SLUB_RED_INACTIVE); in free_debug_processing()
1202 if (object != tail) { in free_debug_processing()
1203 object = get_freepointer(s, object); in free_debug_processing()
1216 slab_fix(s, "Object at 0x%p not freed", object); in free_debug_processing()
1343 struct page *page, void *object) {} in setup_object_debug() argument
1348 struct page *page, void *object, unsigned long addr) { return 0; } in alloc_debug_processing() argument
1358 void *object, u8 val) { return 1; } in check_object() argument
1431 void *object; in slab_free_freelist_hook() local
1441 object = next; in slab_free_freelist_hook()
1442 next = get_freepointer(s, object); in slab_free_freelist_hook()
1449 memset(object, 0, s->object_size); in slab_free_freelist_hook()
1452 memset((char *)object + s->inuse, 0, in slab_free_freelist_hook()
1457 if (!slab_free_hook(s, object)) { in slab_free_freelist_hook()
1459 set_freepointer(s, object, *head); in slab_free_freelist_hook()
1460 *head = object; in slab_free_freelist_hook()
1462 *tail = object; in slab_free_freelist_hook()
1464 } while (object != old_tail); in slab_free_freelist_hook()
1473 void *object) in setup_object() argument
1475 setup_object_debug(s, page, object); in setup_object()
1476 object = kasan_init_slab_obj(s, object); in setup_object()
1478 kasan_unpoison_object_data(s, object); in setup_object()
1479 s->ctor(object); in setup_object()
1480 kasan_poison_object_data(s, object); in setup_object()
1482 return object; in setup_object()
1839 void *object = NULL; in get_partial_node() local
1859 t = acquire_slab(s, n, page, object == NULL, &objects); in get_partial_node()
1864 if (!object) { in get_partial_node()
1867 object = t; in get_partial_node()
1878 return object; in get_partial_node()
1892 void *object; in get_any_partial() local
1927 object = get_partial_node(s, n, c, flags); in get_any_partial()
1928 if (object) { in get_any_partial()
1936 return object; in get_any_partial()
1951 void *object; in get_partial() local
1959 object = get_partial_node(s, get_node(s, searchnode), c, flags); in get_partial()
1960 if (object || node != NUMA_NO_NODE) in get_partial()
1961 return object; in get_partial()
2677 void *object; in slab_alloc_node() local
2719 object = c->freelist; in slab_alloc_node()
2721 if (unlikely(!object || !node_match(page, node))) { in slab_alloc_node()
2722 object = __slab_alloc(s, gfpflags, node, addr, c); in slab_alloc_node()
2725 void *next_object = get_freepointer_safe(s, object); in slab_alloc_node()
2743 object, tid, in slab_alloc_node()
2753 maybe_wipe_obj_freeptr(s, object); in slab_alloc_node()
2755 if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object) in slab_alloc_node()
2756 memset(object, 0, s->object_size); in slab_alloc_node()
2758 slab_post_alloc_hook(s, gfpflags, 1, &object); in slab_alloc_node()
2760 return object; in slab_alloc_node()
3051 void *object; in build_detached_freelist() local
3058 object = p[--size]; in build_detached_freelist()
3060 } while (!object && size); in build_detached_freelist()
3062 if (!object) in build_detached_freelist()
3065 page = virt_to_head_page(object); in build_detached_freelist()
3070 kfree_hook(object); in build_detached_freelist()
3078 df->s = cache_from_obj(s, object); /* Support for memcg */ in build_detached_freelist()
3083 set_freepointer(df->s, object, NULL); in build_detached_freelist()
3084 df->tail = object; in build_detached_freelist()
3085 df->freelist = object; in build_detached_freelist()
3090 object = p[--size]; in build_detached_freelist()
3091 if (!object) in build_detached_freelist()
3095 if (df->page == virt_to_head_page(object)) { in build_detached_freelist()
3097 set_freepointer(df->s, object, df->freelist); in build_detached_freelist()
3098 df->freelist = object; in build_detached_freelist()
3154 void *object = c->freelist; in kmem_cache_alloc_bulk() local
3156 if (unlikely(!object)) { in kmem_cache_alloc_bulk()
3171 c->freelist = get_freepointer(s, object); in kmem_cache_alloc_bulk()
3172 p[i] = object; in kmem_cache_alloc_bulk()
3917 size_t __ksize(const void *object) in __ksize() argument
3921 if (unlikely(object == ZERO_SIZE_PTR)) in __ksize()
3924 page = virt_to_head_page(object); in __ksize()
3938 void *object = (void *)x; in kfree() local
3950 kfree_hook(object); in kfree()
3956 slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_); in kfree()