Lines Matching refs:objp
330 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) in dbg_redzone1() argument
333 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
337 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) in dbg_redzone2() argument
341 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
344 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
348 static void **dbg_userword(struct kmem_cache *cachep, void *objp) in dbg_userword() argument
351 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
357 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
358 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
359 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) argument
597 struct page *page, void *objp) in cache_free_pfmemalloc() argument
607 free_block(cachep, &objp, 1, page_node, &list); in cache_free_pfmemalloc()
651 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
795 static int __cache_free_alien(struct kmem_cache *cachep, void *objp, in __cache_free_alien() argument
813 ac->entry[ac->avail++] = objp; in __cache_free_alien()
819 free_block(cachep, &objp, 1, page_node, &list); in __cache_free_alien()
826 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
828 int page_node = page_to_nid(virt_to_page(objp)); in cache_free_alien()
837 return __cache_free_alien(cachep, objp, node, page_node); in cache_free_alien()
1551 static void slab_kernel_map(struct kmem_cache *cachep, void *objp, in slab_kernel_map() argument
1558 store_stackinfo(cachep, objp, caller); in slab_kernel_map()
1560 kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map); in slab_kernel_map()
1564 static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp, in slab_kernel_map() argument
1610 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) in print_objinfo() argument
1617 *dbg_redzone1(cachep, objp), in print_objinfo()
1618 *dbg_redzone2(cachep, objp)); in print_objinfo()
1623 *dbg_userword(cachep, objp), in print_objinfo()
1624 *dbg_userword(cachep, objp)); in print_objinfo()
1626 realobj = (char *)objp + obj_offset(cachep); in print_objinfo()
1637 static void check_poison_obj(struct kmem_cache *cachep, void *objp) in check_poison_obj() argument
1646 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1661 print_objinfo(cachep, objp, 0); in check_poison_obj()
1680 struct page *page = virt_to_head_page(objp); in check_poison_obj()
1683 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1685 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1686 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1688 print_objinfo(cachep, objp, 2); in check_poison_obj()
1691 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1692 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1694 print_objinfo(cachep, objp, 2); in check_poison_obj()
1712 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck() local
1715 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1716 slab_kernel_map(cachep, objp, 1, 0); in slab_destroy_debugcheck()
1719 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1721 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
2434 void *objp = index_to_obj(cachep, page, i); in cache_init_objs_debug() local
2437 *dbg_userword(cachep, objp) = NULL; in cache_init_objs_debug()
2440 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2441 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2450 objp + obj_offset(cachep)); in cache_init_objs_debug()
2451 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs_debug()
2453 cachep, objp + obj_offset(cachep)); in cache_init_objs_debug()
2457 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2459 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2464 poison_obj(cachep, objp, POISON_FREE); in cache_init_objs_debug()
2465 slab_kernel_map(cachep, objp, 0, 0); in cache_init_objs_debug()
2586 void *objp; in cache_init_objs() local
2600 objp = index_to_obj(cachep, page, i); in cache_init_objs()
2601 kasan_init_slab_obj(cachep, objp); in cache_init_objs()
2605 kasan_unpoison_object_data(cachep, objp); in cache_init_objs()
2606 cachep->ctor(objp); in cache_init_objs()
2607 kasan_poison_object_data(cachep, objp); in cache_init_objs()
2617 void *objp; in slab_get_obj() local
2619 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2627 return objp; in slab_get_obj()
2631 struct page *page, void *objp) in slab_put_obj() argument
2633 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2641 cachep->name, objp); in slab_put_obj()
2648 page->freelist = objp + obj_offset(cachep); in slab_put_obj()
2776 static void kfree_debugcheck(const void *objp) in kfree_debugcheck() argument
2778 if (!virt_addr_valid(objp)) { in kfree_debugcheck()
2780 (unsigned long)objp); in kfree_debugcheck()
2807 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, in cache_free_debugcheck() argument
2813 BUG_ON(virt_to_cache(objp) != cachep); in cache_free_debugcheck()
2815 objp -= obj_offset(cachep); in cache_free_debugcheck()
2816 kfree_debugcheck(objp); in cache_free_debugcheck()
2817 page = virt_to_head_page(objp); in cache_free_debugcheck()
2820 verify_redzone_free(cachep, objp); in cache_free_debugcheck()
2821 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2822 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2826 *dbg_userword(cachep, objp) = (void *)caller; in cache_free_debugcheck()
2829 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2832 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2835 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2836 slab_kernel_map(cachep, objp, 0, caller); in cache_free_debugcheck()
2838 return objp; in cache_free_debugcheck()
2843 #define cache_free_debugcheck(x,objp,z) (objp) argument
2851 void *objp; in fixup_objfreelist_debug() local
2854 objp = next - obj_offset(cachep); in fixup_objfreelist_debug()
2856 poison_obj(cachep, objp, POISON_FREE); in fixup_objfreelist_debug()
2873 void **objp = page->freelist; in fixup_slab_list() local
2875 *objp = *list; in fixup_slab_list()
2876 *list = objp; in fixup_slab_list()
3087 gfp_t flags, void *objp, unsigned long caller) in cache_alloc_debugcheck_after() argument
3089 if (!objp) in cache_alloc_debugcheck_after()
3090 return objp; in cache_alloc_debugcheck_after()
3092 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
3093 slab_kernel_map(cachep, objp, 1, 0); in cache_alloc_debugcheck_after()
3094 poison_obj(cachep, objp, POISON_INUSE); in cache_alloc_debugcheck_after()
3097 *dbg_userword(cachep, objp) = (void *)caller; in cache_alloc_debugcheck_after()
3100 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || in cache_alloc_debugcheck_after()
3101 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { in cache_alloc_debugcheck_after()
3104 objp, *dbg_redzone1(cachep, objp), in cache_alloc_debugcheck_after()
3105 *dbg_redzone2(cachep, objp)); in cache_alloc_debugcheck_after()
3107 *dbg_redzone1(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3108 *dbg_redzone2(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3111 objp += obj_offset(cachep); in cache_alloc_debugcheck_after()
3113 cachep->ctor(objp); in cache_alloc_debugcheck_after()
3115 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { in cache_alloc_debugcheck_after()
3117 objp, (int)ARCH_SLAB_MINALIGN); in cache_alloc_debugcheck_after()
3119 return objp; in cache_alloc_debugcheck_after()
3122 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp) argument
3127 void *objp; in ____cache_alloc() local
3135 objp = ac->entry[--ac->avail]; in ____cache_alloc()
3142 objp = cache_alloc_refill(cachep, flags); in ____cache_alloc()
3155 if (objp) in ____cache_alloc()
3157 return objp; in ____cache_alloc()
3357 void *objp; in __do_cache_alloc() local
3360 objp = alternate_node_alloc(cache, flags); in __do_cache_alloc()
3361 if (objp) in __do_cache_alloc()
3364 objp = ____cache_alloc(cache, flags); in __do_cache_alloc()
3370 if (!objp) in __do_cache_alloc()
3371 objp = ____cache_alloc_node(cache, flags, numa_mem_id()); in __do_cache_alloc()
3374 return objp; in __do_cache_alloc()
3390 void *objp; in slab_alloc() local
3399 objp = __do_cache_alloc(cachep, flags); in slab_alloc()
3401 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); in slab_alloc()
3402 prefetchw(objp); in slab_alloc()
3404 if (unlikely(flags & __GFP_ZERO) && objp) in slab_alloc()
3405 memset(objp, 0, cachep->object_size); in slab_alloc()
3407 slab_post_alloc_hook(cachep, flags, 1, &objp); in slab_alloc()
3408 return objp; in slab_alloc()
3425 void *objp; in free_block() local
3428 objp = objpp[i]; in free_block()
3430 page = virt_to_head_page(objp); in free_block()
3433 slab_put_obj(cachep, page, objp); in free_block()
3507 static inline void __cache_free(struct kmem_cache *cachep, void *objp, in __cache_free() argument
3511 if (kasan_slab_free(cachep, objp)) in __cache_free()
3514 ___cache_free(cachep, objp, caller); in __cache_free()
3517 void ___cache_free(struct kmem_cache *cachep, void *objp, in ___cache_free() argument
3523 kmemleak_free_recursive(objp, cachep->flags); in ___cache_free()
3524 objp = cache_free_debugcheck(cachep, objp, caller); in ___cache_free()
3526 kmemcheck_slab_free(cachep, objp, cachep->object_size); in ___cache_free()
3535 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) in ___cache_free()
3546 struct page *page = virt_to_head_page(objp); in ___cache_free()
3549 cache_free_pfmemalloc(cachep, page, objp); in ___cache_free()
3554 ac->entry[ac->avail++] = objp; in ___cache_free()
3600 void *objp = __do_cache_alloc(s, flags); in kmem_cache_alloc_bulk() local
3602 if (unlikely(!objp)) in kmem_cache_alloc_bulk()
3604 p[i] = objp; in kmem_cache_alloc_bulk()
3760 void kmem_cache_free(struct kmem_cache *cachep, void *objp) in kmem_cache_free() argument
3763 cachep = cache_from_obj(cachep, objp); in kmem_cache_free()
3768 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3770 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3771 __cache_free(cachep, objp, _RET_IP_); in kmem_cache_free()
3774 trace_kmem_cache_free(_RET_IP_, objp); in kmem_cache_free()
3785 void *objp = p[i]; in kmem_cache_free_bulk() local
3788 s = virt_to_cache(objp); in kmem_cache_free_bulk()
3790 s = cache_from_obj(orig_s, objp); in kmem_cache_free_bulk()
3792 debug_check_no_locks_freed(objp, s->object_size); in kmem_cache_free_bulk()
3794 debug_check_no_obj_freed(objp, s->object_size); in kmem_cache_free_bulk()
3796 __cache_free(s, objp, _RET_IP_); in kmem_cache_free_bulk()
3813 void kfree(const void *objp) in kfree() argument
3818 trace_kfree(_RET_IP_, objp); in kfree()
3820 if (unlikely(ZERO_OR_NULL_PTR(objp))) in kfree()
3823 kfree_debugcheck(objp); in kfree()
3824 c = virt_to_cache(objp); in kfree()
3825 debug_check_no_locks_freed(objp, c->object_size); in kfree()
3827 debug_check_no_obj_freed(objp, c->object_size); in kfree()
3828 __cache_free(c, (void *)objp, _RET_IP_); in kfree()
4486 size_t ksize(const void *objp) in ksize() argument
4490 BUG_ON(!objp); in ksize()
4491 if (unlikely(objp == ZERO_SIZE_PTR)) in ksize()
4494 size = virt_to_cache(objp)->object_size; in ksize()
4498 kasan_unpoison_shadow(objp, size); in ksize()