Lines Matching refs:cachep
211 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
213 static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
214 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
217 static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
219 static inline void fixup_slab_list(struct kmem_cache *cachep,
241 #define MAKE_LIST(cachep, listp, slab, nodeid) \ argument
244 list_splice(&get_node(cachep, nodeid)->slab, listp); \
247 #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ argument
249 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
250 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
251 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
327 static int obj_offset(struct kmem_cache *cachep) in obj_offset() argument
329 return cachep->obj_offset; in obj_offset()
332 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) in dbg_redzone1() argument
334 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone1()
335 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
339 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) in dbg_redzone2() argument
341 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone2()
342 if (cachep->flags & SLAB_STORE_USER) in dbg_redzone2()
343 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
346 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
350 static void **dbg_userword(struct kmem_cache *cachep, void *objp) in dbg_userword() argument
352 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); in dbg_userword()
353 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
359 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
360 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
361 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) argument
392 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) in cpu_cache_get() argument
394 return this_cpu_ptr(cachep->cpu_cache); in cpu_cache_get()
436 #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg) argument
438 static void __slab_error(const char *function, struct kmem_cache *cachep, in __slab_error() argument
442 function, cachep->name, msg); in __slab_error()
551 static noinline void cache_free_pfmemalloc(struct kmem_cache *cachep, in cache_free_pfmemalloc() argument
559 n = get_node(cachep, page_node); in cache_free_pfmemalloc()
562 free_block(cachep, &objp, 1, page_node, &list); in cache_free_pfmemalloc()
565 slabs_destroy(cachep, &list); in cache_free_pfmemalloc()
593 #define drain_alien_cache(cachep, alien) do { } while (0) argument
594 #define reap_alien(cachep, n) do { } while (0) argument
606 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
611 static inline void *alternate_node_alloc(struct kmem_cache *cachep, in alternate_node_alloc() argument
617 static inline void *____cache_alloc_node(struct kmem_cache *cachep, in ____cache_alloc_node() argument
684 static void __drain_alien_cache(struct kmem_cache *cachep, in __drain_alien_cache() argument
688 struct kmem_cache_node *n = get_node(cachep, node); in __drain_alien_cache()
700 free_block(cachep, ac->entry, ac->avail, node, list); in __drain_alien_cache()
709 static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) in reap_alien() argument
722 __drain_alien_cache(cachep, ac, node, &list); in reap_alien()
724 slabs_destroy(cachep, &list); in reap_alien()
730 static void drain_alien_cache(struct kmem_cache *cachep, in drain_alien_cache() argument
745 __drain_alien_cache(cachep, ac, i, &list); in drain_alien_cache()
747 slabs_destroy(cachep, &list); in drain_alien_cache()
752 static int __cache_free_alien(struct kmem_cache *cachep, void *objp, in __cache_free_alien() argument
760 n = get_node(cachep, node); in __cache_free_alien()
761 STATS_INC_NODEFREES(cachep); in __cache_free_alien()
767 STATS_INC_ACOVERFLOW(cachep); in __cache_free_alien()
768 __drain_alien_cache(cachep, ac, page_node, &list); in __cache_free_alien()
772 slabs_destroy(cachep, &list); in __cache_free_alien()
774 n = get_node(cachep, page_node); in __cache_free_alien()
776 free_block(cachep, &objp, 1, page_node, &list); in __cache_free_alien()
778 slabs_destroy(cachep, &list); in __cache_free_alien()
783 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
794 return __cache_free_alien(cachep, objp, node, page_node); in cache_free_alien()
807 static int init_cache_node(struct kmem_cache *cachep, int node, gfp_t gfp) in init_cache_node() argument
816 n = get_node(cachep, node); in init_cache_node()
819 n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + in init_cache_node()
820 cachep->num; in init_cache_node()
832 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in init_cache_node()
835 (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; in init_cache_node()
842 cachep->node[node] = n; in init_cache_node()
860 struct kmem_cache *cachep; in init_cache_node_node() local
862 list_for_each_entry(cachep, &slab_caches, list) { in init_cache_node_node()
863 ret = init_cache_node(cachep, node, GFP_KERNEL); in init_cache_node_node()
872 static int setup_kmem_cache_node(struct kmem_cache *cachep, in setup_kmem_cache_node() argument
883 new_alien = alloc_alien_cache(node, cachep->limit, gfp); in setup_kmem_cache_node()
888 if (cachep->shared) { in setup_kmem_cache_node()
890 cachep->shared * cachep->batchcount, 0xbaadf00d, gfp); in setup_kmem_cache_node()
895 ret = init_cache_node(cachep, node, gfp); in setup_kmem_cache_node()
899 n = get_node(cachep, node); in setup_kmem_cache_node()
902 free_block(cachep, n->shared->entry, in setup_kmem_cache_node()
919 slabs_destroy(cachep, &list); in setup_kmem_cache_node()
942 struct kmem_cache *cachep; in cpuup_canceled() local
947 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_canceled()
953 n = get_node(cachep, node); in cpuup_canceled()
960 n->free_limit -= cachep->batchcount; in cpuup_canceled()
963 nc = per_cpu_ptr(cachep->cpu_cache, cpu); in cpuup_canceled()
964 free_block(cachep, nc->entry, nc->avail, node, &list); in cpuup_canceled()
974 free_block(cachep, shared->entry, in cpuup_canceled()
986 drain_alien_cache(cachep, alien); in cpuup_canceled()
991 slabs_destroy(cachep, &list); in cpuup_canceled()
998 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_canceled()
999 n = get_node(cachep, node); in cpuup_canceled()
1002 drain_freelist(cachep, n, INT_MAX); in cpuup_canceled()
1008 struct kmem_cache *cachep; in cpuup_prepare() local
1026 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_prepare()
1027 err = setup_kmem_cache_node(cachep, node, GFP_KERNEL, false); in cpuup_prepare()
1097 struct kmem_cache *cachep; in drain_cache_node_node() local
1100 list_for_each_entry(cachep, &slab_caches, list) { in drain_cache_node_node()
1103 n = get_node(cachep, node); in drain_cache_node_node()
1107 drain_freelist(cachep, n, INT_MAX); in drain_cache_node_node()
1154 static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, in init_list() argument
1168 MAKE_ALL_LISTS(cachep, ptr, nodeid); in init_list()
1169 cachep->node[nodeid] = ptr; in init_list()
1176 static void __init set_up_node(struct kmem_cache *cachep, int index) in set_up_node() argument
1181 cachep->node[node] = &init_kmem_cache_node[index + node]; in set_up_node()
1182 cachep->node[node]->next_reap = jiffies + in set_up_node()
1184 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in set_up_node()
1275 struct kmem_cache *cachep; in kmem_cache_init_late() local
1279 list_for_each_entry(cachep, &slab_caches, list) in kmem_cache_init_late()
1280 if (enable_cpucache(cachep, GFP_NOWAIT)) in kmem_cache_init_late()
1317 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) in slab_out_of_memory() argument
1332 cachep->name, cachep->size, cachep->gfporder); in slab_out_of_memory()
1334 for_each_kmem_cache_node(cachep, node, n) { in slab_out_of_memory()
1345 (total_slabs * cachep->num) - free_objs, in slab_out_of_memory()
1346 total_slabs * cachep->num); in slab_out_of_memory()
1359 static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, in kmem_getpages() argument
1364 flags |= cachep->allocflags; in kmem_getpages()
1366 page = __alloc_pages_node(nodeid, flags, cachep->gfporder); in kmem_getpages()
1368 slab_out_of_memory(cachep, flags, nodeid); in kmem_getpages()
1372 if (charge_slab_page(page, flags, cachep->gfporder, cachep)) { in kmem_getpages()
1373 __free_pages(page, cachep->gfporder); in kmem_getpages()
1388 static void kmem_freepages(struct kmem_cache *cachep, struct page *page) in kmem_freepages() argument
1390 int order = cachep->gfporder; in kmem_freepages()
1400 uncharge_slab_page(page, order, cachep); in kmem_freepages()
1406 struct kmem_cache *cachep; in kmem_rcu_free() local
1410 cachep = page->slab_cache; in kmem_rcu_free()
1412 kmem_freepages(cachep, page); in kmem_rcu_free()
1416 static bool is_debug_pagealloc_cache(struct kmem_cache *cachep) in is_debug_pagealloc_cache() argument
1418 if (debug_pagealloc_enabled_static() && OFF_SLAB(cachep) && in is_debug_pagealloc_cache()
1419 (cachep->size % PAGE_SIZE) == 0) in is_debug_pagealloc_cache()
1426 static void slab_kernel_map(struct kmem_cache *cachep, void *objp, int map) in slab_kernel_map() argument
1428 if (!is_debug_pagealloc_cache(cachep)) in slab_kernel_map()
1431 kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map); in slab_kernel_map()
1435 static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp, in slab_kernel_map() argument
1440 static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) in poison_obj() argument
1442 int size = cachep->object_size; in poison_obj()
1443 addr = &((char *)addr)[obj_offset(cachep)]; in poison_obj()
1481 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) in print_objinfo() argument
1486 if (cachep->flags & SLAB_RED_ZONE) { in print_objinfo()
1488 *dbg_redzone1(cachep, objp), in print_objinfo()
1489 *dbg_redzone2(cachep, objp)); in print_objinfo()
1492 if (cachep->flags & SLAB_STORE_USER) in print_objinfo()
1493 pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp)); in print_objinfo()
1494 realobj = (char *)objp + obj_offset(cachep); in print_objinfo()
1495 size = cachep->object_size; in print_objinfo()
1505 static void check_poison_obj(struct kmem_cache *cachep, void *objp) in check_poison_obj() argument
1511 if (is_debug_pagealloc_cache(cachep)) in check_poison_obj()
1514 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1515 size = cachep->object_size; in check_poison_obj()
1527 print_tainted(), cachep->name, in check_poison_obj()
1529 print_objinfo(cachep, objp, 0); in check_poison_obj()
1551 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1553 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1554 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1556 print_objinfo(cachep, objp, 2); in check_poison_obj()
1558 if (objnr + 1 < cachep->num) { in check_poison_obj()
1559 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1560 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1562 print_objinfo(cachep, objp, 2); in check_poison_obj()
1569 static void slab_destroy_debugcheck(struct kmem_cache *cachep, in slab_destroy_debugcheck() argument
1574 if (OBJFREELIST_SLAB(cachep) && cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1575 poison_obj(cachep, page->freelist - obj_offset(cachep), in slab_destroy_debugcheck()
1579 for (i = 0; i < cachep->num; i++) { in slab_destroy_debugcheck()
1580 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck()
1582 if (cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1583 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1584 slab_kernel_map(cachep, objp, 1); in slab_destroy_debugcheck()
1586 if (cachep->flags & SLAB_RED_ZONE) { in slab_destroy_debugcheck()
1587 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1588 slab_error(cachep, "start of a freed object was overwritten"); in slab_destroy_debugcheck()
1589 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1590 slab_error(cachep, "end of a freed object was overwritten"); in slab_destroy_debugcheck()
1595 static void slab_destroy_debugcheck(struct kmem_cache *cachep, in slab_destroy_debugcheck() argument
1610 static void slab_destroy(struct kmem_cache *cachep, struct page *page) in slab_destroy() argument
1615 slab_destroy_debugcheck(cachep, page); in slab_destroy()
1616 if (unlikely(cachep->flags & SLAB_TYPESAFE_BY_RCU)) in slab_destroy()
1619 kmem_freepages(cachep, page); in slab_destroy()
1625 if (OFF_SLAB(cachep)) in slab_destroy()
1626 kmem_cache_free(cachep->freelist_cache, freelist); in slab_destroy()
1629 static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list) in slabs_destroy() argument
1635 slab_destroy(cachep, page); in slabs_destroy()
1653 static size_t calculate_slab_order(struct kmem_cache *cachep, in calculate_slab_order() argument
1688 if (freelist_cache->size > cachep->size / 2) in calculate_slab_order()
1693 cachep->num = num; in calculate_slab_order()
1694 cachep->gfporder = gfporder; in calculate_slab_order()
1722 struct kmem_cache *cachep, int entries, int batchcount) in alloc_kmem_cache_cpus() argument
1742 static int __ref setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) in setup_cpu_cache() argument
1745 return enable_cpucache(cachep, gfp); in setup_cpu_cache()
1747 cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1); in setup_cpu_cache()
1748 if (!cachep->cpu_cache) in setup_cpu_cache()
1756 set_up_node(cachep, SIZE_NODE); in setup_cpu_cache()
1761 cachep->node[node] = kmalloc_node( in setup_cpu_cache()
1763 BUG_ON(!cachep->node[node]); in setup_cpu_cache()
1764 kmem_cache_node_init(cachep->node[node]); in setup_cpu_cache()
1768 cachep->node[numa_mem_id()]->next_reap = in setup_cpu_cache()
1770 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in setup_cpu_cache()
1772 cpu_cache_get(cachep)->avail = 0; in setup_cpu_cache()
1773 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
1774 cpu_cache_get(cachep)->batchcount = 1; in setup_cpu_cache()
1775 cpu_cache_get(cachep)->touched = 0; in setup_cpu_cache()
1776 cachep->batchcount = 1; in setup_cpu_cache()
1777 cachep->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
1792 struct kmem_cache *cachep; in __kmem_cache_alias() local
1794 cachep = find_mergeable(size, align, flags, name, ctor); in __kmem_cache_alias()
1795 if (cachep) { in __kmem_cache_alias()
1796 cachep->refcount++; in __kmem_cache_alias()
1802 cachep->object_size = max_t(int, cachep->object_size, size); in __kmem_cache_alias()
1804 return cachep; in __kmem_cache_alias()
1807 static bool set_objfreelist_slab_cache(struct kmem_cache *cachep, in set_objfreelist_slab_cache() argument
1812 cachep->num = 0; in set_objfreelist_slab_cache()
1819 if (unlikely(slab_want_init_on_free(cachep))) in set_objfreelist_slab_cache()
1822 if (cachep->ctor || flags & SLAB_TYPESAFE_BY_RCU) in set_objfreelist_slab_cache()
1825 left = calculate_slab_order(cachep, size, in set_objfreelist_slab_cache()
1827 if (!cachep->num) in set_objfreelist_slab_cache()
1830 if (cachep->num * sizeof(freelist_idx_t) > cachep->object_size) in set_objfreelist_slab_cache()
1833 cachep->colour = left / cachep->colour_off; in set_objfreelist_slab_cache()
1838 static bool set_off_slab_cache(struct kmem_cache *cachep, in set_off_slab_cache() argument
1843 cachep->num = 0; in set_off_slab_cache()
1856 left = calculate_slab_order(cachep, size, flags | CFLGS_OFF_SLAB); in set_off_slab_cache()
1857 if (!cachep->num) in set_off_slab_cache()
1864 if (left >= cachep->num * sizeof(freelist_idx_t)) in set_off_slab_cache()
1867 cachep->colour = left / cachep->colour_off; in set_off_slab_cache()
1872 static bool set_on_slab_cache(struct kmem_cache *cachep, in set_on_slab_cache() argument
1877 cachep->num = 0; in set_on_slab_cache()
1879 left = calculate_slab_order(cachep, size, flags); in set_on_slab_cache()
1880 if (!cachep->num) in set_on_slab_cache()
1883 cachep->colour = left / cachep->colour_off; in set_on_slab_cache()
1911 int __kmem_cache_create(struct kmem_cache *cachep, slab_flags_t flags) in __kmem_cache_create() argument
1916 unsigned int size = cachep->size; in __kmem_cache_create()
1949 if (ralign < cachep->align) { in __kmem_cache_create()
1950 ralign = cachep->align; in __kmem_cache_create()
1958 cachep->align = ralign; in __kmem_cache_create()
1959 cachep->colour_off = cache_line_size(); in __kmem_cache_create()
1961 if (cachep->colour_off < cachep->align) in __kmem_cache_create()
1962 cachep->colour_off = cachep->align; in __kmem_cache_create()
1977 cachep->obj_offset += sizeof(unsigned long long); in __kmem_cache_create()
1992 kasan_cache_create(cachep, &size, &flags); in __kmem_cache_create()
1994 size = ALIGN(size, cachep->align); in __kmem_cache_create()
2000 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align); in __kmem_cache_create()
2011 size >= 256 && cachep->object_size > cache_line_size()) { in __kmem_cache_create()
2015 if (set_off_slab_cache(cachep, tmp_size, flags)) { in __kmem_cache_create()
2017 cachep->obj_offset += tmp_size - size; in __kmem_cache_create()
2025 if (set_objfreelist_slab_cache(cachep, size, flags)) { in __kmem_cache_create()
2030 if (set_off_slab_cache(cachep, size, flags)) { in __kmem_cache_create()
2035 if (set_on_slab_cache(cachep, size, flags)) in __kmem_cache_create()
2041 cachep->freelist_size = cachep->num * sizeof(freelist_idx_t); in __kmem_cache_create()
2042 cachep->flags = flags; in __kmem_cache_create()
2043 cachep->allocflags = __GFP_COMP; in __kmem_cache_create()
2045 cachep->allocflags |= GFP_DMA; in __kmem_cache_create()
2047 cachep->allocflags |= GFP_DMA32; in __kmem_cache_create()
2049 cachep->allocflags |= __GFP_RECLAIMABLE; in __kmem_cache_create()
2050 cachep->size = size; in __kmem_cache_create()
2051 cachep->reciprocal_buffer_size = reciprocal_value(size); in __kmem_cache_create()
2060 (cachep->flags & SLAB_POISON) && in __kmem_cache_create()
2061 is_debug_pagealloc_cache(cachep)) in __kmem_cache_create()
2062 cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); in __kmem_cache_create()
2065 if (OFF_SLAB(cachep)) { in __kmem_cache_create()
2066 cachep->freelist_cache = in __kmem_cache_create()
2067 kmalloc_slab(cachep->freelist_size, 0u); in __kmem_cache_create()
2070 err = setup_cpu_cache(cachep, gfp); in __kmem_cache_create()
2072 __kmem_cache_release(cachep); in __kmem_cache_create()
2095 static void check_spinlock_acquired(struct kmem_cache *cachep) in check_spinlock_acquired() argument
2099 assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock); in check_spinlock_acquired()
2103 static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) in check_spinlock_acquired_node() argument
2107 assert_spin_locked(&get_node(cachep, node)->list_lock); in check_spinlock_acquired_node()
2119 static void drain_array_locked(struct kmem_cache *cachep, struct array_cache *ac, in drain_array_locked() argument
2131 free_block(cachep, ac->entry, tofree, node, list); in drain_array_locked()
2138 struct kmem_cache *cachep = arg; in do_drain() local
2145 ac = cpu_cache_get(cachep); in do_drain()
2146 n = get_node(cachep, node); in do_drain()
2148 free_block(cachep, ac->entry, ac->avail, node, &list); in do_drain()
2150 slabs_destroy(cachep, &list); in do_drain()
2154 static void drain_cpu_caches(struct kmem_cache *cachep) in drain_cpu_caches() argument
2160 on_each_cpu(do_drain, cachep, 1); in drain_cpu_caches()
2162 for_each_kmem_cache_node(cachep, node, n) in drain_cpu_caches()
2164 drain_alien_cache(cachep, n->alien); in drain_cpu_caches()
2166 for_each_kmem_cache_node(cachep, node, n) { in drain_cpu_caches()
2168 drain_array_locked(cachep, n->shared, node, true, &list); in drain_cpu_caches()
2171 slabs_destroy(cachep, &list); in drain_cpu_caches()
2227 int __kmem_cache_shrink(struct kmem_cache *cachep) in __kmem_cache_shrink() argument
2233 drain_cpu_caches(cachep); in __kmem_cache_shrink()
2236 for_each_kmem_cache_node(cachep, node, n) { in __kmem_cache_shrink()
2237 drain_freelist(cachep, n, INT_MAX); in __kmem_cache_shrink()
2246 void __kmemcg_cache_deactivate(struct kmem_cache *cachep) in __kmemcg_cache_deactivate() argument
2248 __kmem_cache_shrink(cachep); in __kmemcg_cache_deactivate()
2256 int __kmem_cache_shutdown(struct kmem_cache *cachep) in __kmem_cache_shutdown() argument
2258 return __kmem_cache_shrink(cachep); in __kmem_cache_shutdown()
2261 void __kmem_cache_release(struct kmem_cache *cachep) in __kmem_cache_release() argument
2266 cache_random_seq_destroy(cachep); in __kmem_cache_release()
2268 free_percpu(cachep->cpu_cache); in __kmem_cache_release()
2271 for_each_kmem_cache_node(cachep, i, n) { in __kmem_cache_release()
2275 cachep->node[i] = NULL; in __kmem_cache_release()
2293 static void *alloc_slabmgmt(struct kmem_cache *cachep, in alloc_slabmgmt() argument
2303 if (OBJFREELIST_SLAB(cachep)) in alloc_slabmgmt()
2305 else if (OFF_SLAB(cachep)) { in alloc_slabmgmt()
2307 freelist = kmem_cache_alloc_node(cachep->freelist_cache, in alloc_slabmgmt()
2313 freelist = addr + (PAGE_SIZE << cachep->gfporder) - in alloc_slabmgmt()
2314 cachep->freelist_size; in alloc_slabmgmt()
2331 static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page) in cache_init_objs_debug() argument
2336 for (i = 0; i < cachep->num; i++) { in cache_init_objs_debug()
2337 void *objp = index_to_obj(cachep, page, i); in cache_init_objs_debug()
2339 if (cachep->flags & SLAB_STORE_USER) in cache_init_objs_debug()
2340 *dbg_userword(cachep, objp) = NULL; in cache_init_objs_debug()
2342 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs_debug()
2343 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2344 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2351 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) { in cache_init_objs_debug()
2352 kasan_unpoison_object_data(cachep, in cache_init_objs_debug()
2353 objp + obj_offset(cachep)); in cache_init_objs_debug()
2354 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs_debug()
2356 cachep, objp + obj_offset(cachep)); in cache_init_objs_debug()
2359 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs_debug()
2360 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2361 slab_error(cachep, "constructor overwrote the end of an object"); in cache_init_objs_debug()
2362 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2363 slab_error(cachep, "constructor overwrote the start of an object"); in cache_init_objs_debug()
2366 if (cachep->flags & SLAB_POISON) { in cache_init_objs_debug()
2367 poison_obj(cachep, objp, POISON_FREE); in cache_init_objs_debug()
2368 slab_kernel_map(cachep, objp, 0); in cache_init_objs_debug()
2390 struct kmem_cache *cachep, in freelist_state_initialize() argument
2400 if (!cachep->random_seq) { in freelist_state_initialize()
2404 state->list = cachep->random_seq; in freelist_state_initialize()
2431 static bool shuffle_freelist(struct kmem_cache *cachep, struct page *page) in shuffle_freelist() argument
2433 unsigned int objfreelist = 0, i, rand, count = cachep->num; in shuffle_freelist()
2440 precomputed = freelist_state_initialize(&state, cachep, count); in shuffle_freelist()
2443 if (OBJFREELIST_SLAB(cachep)) { in shuffle_freelist()
2448 page->freelist = index_to_obj(cachep, page, objfreelist) + in shuffle_freelist()
2449 obj_offset(cachep); in shuffle_freelist()
2472 if (OBJFREELIST_SLAB(cachep)) in shuffle_freelist()
2473 set_free_obj(page, cachep->num - 1, objfreelist); in shuffle_freelist()
2478 static inline bool shuffle_freelist(struct kmem_cache *cachep, in shuffle_freelist() argument
2485 static void cache_init_objs(struct kmem_cache *cachep, in cache_init_objs() argument
2492 cache_init_objs_debug(cachep, page); in cache_init_objs()
2495 shuffled = shuffle_freelist(cachep, page); in cache_init_objs()
2497 if (!shuffled && OBJFREELIST_SLAB(cachep)) { in cache_init_objs()
2498 page->freelist = index_to_obj(cachep, page, cachep->num - 1) + in cache_init_objs()
2499 obj_offset(cachep); in cache_init_objs()
2502 for (i = 0; i < cachep->num; i++) { in cache_init_objs()
2503 objp = index_to_obj(cachep, page, i); in cache_init_objs()
2504 objp = kasan_init_slab_obj(cachep, objp); in cache_init_objs()
2507 if (DEBUG == 0 && cachep->ctor) { in cache_init_objs()
2508 kasan_unpoison_object_data(cachep, objp); in cache_init_objs()
2509 cachep->ctor(objp); in cache_init_objs()
2510 kasan_poison_object_data(cachep, objp); in cache_init_objs()
2518 static void *slab_get_obj(struct kmem_cache *cachep, struct page *page) in slab_get_obj() argument
2522 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2528 static void slab_put_obj(struct kmem_cache *cachep, in slab_put_obj() argument
2531 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2536 for (i = page->active; i < cachep->num; i++) { in slab_put_obj()
2539 cachep->name, objp); in slab_put_obj()
2546 page->freelist = objp + obj_offset(cachep); in slab_put_obj()
2567 static struct page *cache_grow_begin(struct kmem_cache *cachep, in cache_grow_begin() argument
2588 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO)); in cache_grow_begin()
2599 page = kmem_getpages(cachep, local_flags, nodeid); in cache_grow_begin()
2604 n = get_node(cachep, page_node); in cache_grow_begin()
2608 if (n->colour_next >= cachep->colour) in cache_grow_begin()
2612 if (offset >= cachep->colour) in cache_grow_begin()
2615 offset *= cachep->colour_off; in cache_grow_begin()
2625 freelist = alloc_slabmgmt(cachep, page, offset, in cache_grow_begin()
2627 if (OFF_SLAB(cachep) && !freelist) in cache_grow_begin()
2630 slab_map_pages(cachep, page, freelist); in cache_grow_begin()
2632 cache_init_objs(cachep, page); in cache_grow_begin()
2640 kmem_freepages(cachep, page); in cache_grow_begin()
2647 static void cache_grow_end(struct kmem_cache *cachep, struct page *page) in cache_grow_end() argument
2658 n = get_node(cachep, page_to_nid(page)); in cache_grow_end()
2666 fixup_slab_list(cachep, n, page, &list); in cache_grow_end()
2668 STATS_INC_GROWN(cachep); in cache_grow_end()
2669 n->free_objects += cachep->num - page->active; in cache_grow_end()
2672 fixup_objfreelist_debug(cachep, &list); in cache_grow_end()
2713 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, in cache_free_debugcheck() argument
2719 BUG_ON(virt_to_cache(objp) != cachep); in cache_free_debugcheck()
2721 objp -= obj_offset(cachep); in cache_free_debugcheck()
2725 if (cachep->flags & SLAB_RED_ZONE) { in cache_free_debugcheck()
2726 verify_redzone_free(cachep, objp); in cache_free_debugcheck()
2727 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2728 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2730 if (cachep->flags & SLAB_STORE_USER) in cache_free_debugcheck()
2731 *dbg_userword(cachep, objp) = (void *)caller; in cache_free_debugcheck()
2733 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2735 BUG_ON(objnr >= cachep->num); in cache_free_debugcheck()
2736 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2738 if (cachep->flags & SLAB_POISON) { in cache_free_debugcheck()
2739 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2740 slab_kernel_map(cachep, objp, 0); in cache_free_debugcheck()
2750 static inline void fixup_objfreelist_debug(struct kmem_cache *cachep, in fixup_objfreelist_debug() argument
2758 objp = next - obj_offset(cachep); in fixup_objfreelist_debug()
2760 poison_obj(cachep, objp, POISON_FREE); in fixup_objfreelist_debug()
2765 static inline void fixup_slab_list(struct kmem_cache *cachep, in fixup_slab_list() argument
2771 if (page->active == cachep->num) { in fixup_slab_list()
2773 if (OBJFREELIST_SLAB(cachep)) { in fixup_slab_list()
2776 if (cachep->flags & SLAB_POISON) { in fixup_slab_list()
2853 static noinline void *cache_alloc_pfmemalloc(struct kmem_cache *cachep, in cache_alloc_pfmemalloc() argument
2870 obj = slab_get_obj(cachep, page); in cache_alloc_pfmemalloc()
2873 fixup_slab_list(cachep, n, page, &list); in cache_alloc_pfmemalloc()
2876 fixup_objfreelist_debug(cachep, &list); in cache_alloc_pfmemalloc()
2885 static __always_inline int alloc_block(struct kmem_cache *cachep, in alloc_block() argument
2892 BUG_ON(page->active >= cachep->num); in alloc_block()
2894 while (page->active < cachep->num && batchcount--) { in alloc_block()
2895 STATS_INC_ALLOCED(cachep); in alloc_block()
2896 STATS_INC_ACTIVE(cachep); in alloc_block()
2897 STATS_SET_HIGH(cachep); in alloc_block()
2899 ac->entry[ac->avail++] = slab_get_obj(cachep, page); in alloc_block()
2905 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) in cache_alloc_refill() argument
2917 ac = cpu_cache_get(cachep); in cache_alloc_refill()
2927 n = get_node(cachep, node); in cache_alloc_refill()
2949 check_spinlock_acquired(cachep); in cache_alloc_refill()
2951 batchcount = alloc_block(cachep, ac, page, batchcount); in cache_alloc_refill()
2952 fixup_slab_list(cachep, n, page, &list); in cache_alloc_refill()
2959 fixup_objfreelist_debug(cachep, &list); in cache_alloc_refill()
2965 void *obj = cache_alloc_pfmemalloc(cachep, n, flags); in cache_alloc_refill()
2971 page = cache_grow_begin(cachep, gfp_exact_node(flags), node); in cache_alloc_refill()
2977 ac = cpu_cache_get(cachep); in cache_alloc_refill()
2979 alloc_block(cachep, ac, page, batchcount); in cache_alloc_refill()
2980 cache_grow_end(cachep, page); in cache_alloc_refill()
2990 static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, in cache_alloc_debugcheck_before() argument
2997 static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, in cache_alloc_debugcheck_after() argument
3000 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO)); in cache_alloc_debugcheck_after()
3003 if (cachep->flags & SLAB_POISON) { in cache_alloc_debugcheck_after()
3004 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
3005 slab_kernel_map(cachep, objp, 1); in cache_alloc_debugcheck_after()
3006 poison_obj(cachep, objp, POISON_INUSE); in cache_alloc_debugcheck_after()
3008 if (cachep->flags & SLAB_STORE_USER) in cache_alloc_debugcheck_after()
3009 *dbg_userword(cachep, objp) = (void *)caller; in cache_alloc_debugcheck_after()
3011 if (cachep->flags & SLAB_RED_ZONE) { in cache_alloc_debugcheck_after()
3012 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || in cache_alloc_debugcheck_after()
3013 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { in cache_alloc_debugcheck_after()
3014 slab_error(cachep, "double free, or memory outside object was overwritten"); in cache_alloc_debugcheck_after()
3016 objp, *dbg_redzone1(cachep, objp), in cache_alloc_debugcheck_after()
3017 *dbg_redzone2(cachep, objp)); in cache_alloc_debugcheck_after()
3019 *dbg_redzone1(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3020 *dbg_redzone2(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3023 objp += obj_offset(cachep); in cache_alloc_debugcheck_after()
3024 if (cachep->ctor && cachep->flags & SLAB_POISON) in cache_alloc_debugcheck_after()
3025 cachep->ctor(objp); in cache_alloc_debugcheck_after()
3037 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) in ____cache_alloc() argument
3044 ac = cpu_cache_get(cachep); in ____cache_alloc()
3049 STATS_INC_ALLOCHIT(cachep); in ____cache_alloc()
3053 STATS_INC_ALLOCMISS(cachep); in ____cache_alloc()
3054 objp = cache_alloc_refill(cachep, flags); in ____cache_alloc()
3059 ac = cpu_cache_get(cachep); in ____cache_alloc()
3079 static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) in alternate_node_alloc() argument
3086 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) in alternate_node_alloc()
3091 return ____cache_alloc_node(cachep, flags, nid_alloc); in alternate_node_alloc()
3170 static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, in ____cache_alloc_node() argument
3179 n = get_node(cachep, nodeid); in ____cache_alloc_node()
3188 check_spinlock_acquired_node(cachep, nodeid); in ____cache_alloc_node()
3190 STATS_INC_NODEALLOCS(cachep); in ____cache_alloc_node()
3191 STATS_INC_ACTIVE(cachep); in ____cache_alloc_node()
3192 STATS_SET_HIGH(cachep); in ____cache_alloc_node()
3194 BUG_ON(page->active == cachep->num); in ____cache_alloc_node()
3196 obj = slab_get_obj(cachep, page); in ____cache_alloc_node()
3199 fixup_slab_list(cachep, n, page, &list); in ____cache_alloc_node()
3202 fixup_objfreelist_debug(cachep, &list); in ____cache_alloc_node()
3207 page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid); in ____cache_alloc_node()
3210 obj = slab_get_obj(cachep, page); in ____cache_alloc_node()
3212 cache_grow_end(cachep, page); in ____cache_alloc_node()
3214 return obj ? obj : fallback_alloc(cachep, flags); in ____cache_alloc_node()
3218 slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, in slab_alloc_node() argument
3226 cachep = slab_pre_alloc_hook(cachep, flags); in slab_alloc_node()
3227 if (unlikely(!cachep)) in slab_alloc_node()
3230 cache_alloc_debugcheck_before(cachep, flags); in slab_alloc_node()
3236 if (unlikely(!get_node(cachep, nodeid))) { in slab_alloc_node()
3238 ptr = fallback_alloc(cachep, flags); in slab_alloc_node()
3249 ptr = ____cache_alloc(cachep, flags); in slab_alloc_node()
3254 ptr = ____cache_alloc_node(cachep, flags, nodeid); in slab_alloc_node()
3257 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); in slab_alloc_node()
3259 if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr) in slab_alloc_node()
3260 memset(ptr, 0, cachep->object_size); in slab_alloc_node()
3262 slab_post_alloc_hook(cachep, flags, 1, &ptr); in slab_alloc_node()
3291 __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) in __do_cache_alloc() argument
3293 return ____cache_alloc(cachep, flags); in __do_cache_alloc()
3299 slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) in slab_alloc() argument
3305 cachep = slab_pre_alloc_hook(cachep, flags); in slab_alloc()
3306 if (unlikely(!cachep)) in slab_alloc()
3309 cache_alloc_debugcheck_before(cachep, flags); in slab_alloc()
3311 objp = __do_cache_alloc(cachep, flags); in slab_alloc()
3313 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); in slab_alloc()
3316 if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp) in slab_alloc()
3317 memset(objp, 0, cachep->object_size); in slab_alloc()
3319 slab_post_alloc_hook(cachep, flags, 1, &objp); in slab_alloc()
3327 static void free_block(struct kmem_cache *cachep, void **objpp, in free_block() argument
3331 struct kmem_cache_node *n = get_node(cachep, node); in free_block()
3344 check_spinlock_acquired_node(cachep, node); in free_block()
3345 slab_put_obj(cachep, page, objp); in free_block()
3346 STATS_DEC_ACTIVE(cachep); in free_block()
3362 n->free_objects -= cachep->num; in free_block()
3371 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) in cache_flusharray() argument
3381 n = get_node(cachep, node); in cache_flusharray()
3396 free_block(cachep, ac->entry, batchcount, node, &list); in cache_flusharray()
3408 STATS_SET_FREEABLE(cachep, i); in cache_flusharray()
3412 slabs_destroy(cachep, &list); in cache_flusharray()
3421 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, in __cache_free() argument
3425 if (kasan_slab_free(cachep, objp, _RET_IP_)) in __cache_free()
3428 ___cache_free(cachep, objp, caller); in __cache_free()
3431 void ___cache_free(struct kmem_cache *cachep, void *objp, in ___cache_free() argument
3434 struct array_cache *ac = cpu_cache_get(cachep); in ___cache_free()
3437 if (unlikely(slab_want_init_on_free(cachep))) in ___cache_free()
3438 memset(objp, 0, cachep->object_size); in ___cache_free()
3439 kmemleak_free_recursive(objp, cachep->flags); in ___cache_free()
3440 objp = cache_free_debugcheck(cachep, objp, caller); in ___cache_free()
3449 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) in ___cache_free()
3453 STATS_INC_FREEHIT(cachep); in ___cache_free()
3455 STATS_INC_FREEMISS(cachep); in ___cache_free()
3456 cache_flusharray(cachep, ac); in ___cache_free()
3463 cache_free_pfmemalloc(cachep, page, objp); in ___cache_free()
3481 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) in kmem_cache_alloc() argument
3483 void *ret = slab_alloc(cachep, flags, _RET_IP_); in kmem_cache_alloc()
3486 cachep->object_size, cachep->size, flags); in kmem_cache_alloc()
3544 kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) in kmem_cache_alloc_trace() argument
3548 ret = slab_alloc(cachep, flags, _RET_IP_); in kmem_cache_alloc_trace()
3550 ret = kasan_kmalloc(cachep, ret, size, flags); in kmem_cache_alloc_trace()
3552 size, cachep->size, flags); in kmem_cache_alloc_trace()
3572 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) in kmem_cache_alloc_node() argument
3574 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); in kmem_cache_alloc_node()
3577 cachep->object_size, cachep->size, in kmem_cache_alloc_node()
3585 void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, in kmem_cache_alloc_node_trace() argument
3592 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); in kmem_cache_alloc_node_trace()
3594 ret = kasan_kmalloc(cachep, ret, size, flags); in kmem_cache_alloc_node_trace()
3596 size, cachep->size, in kmem_cache_alloc_node_trace()
3606 struct kmem_cache *cachep; in __do_kmalloc_node() local
3611 cachep = kmalloc_slab(size, flags); in __do_kmalloc_node()
3612 if (unlikely(ZERO_OR_NULL_PTR(cachep))) in __do_kmalloc_node()
3613 return cachep; in __do_kmalloc_node()
3614 ret = kmem_cache_alloc_node_trace(cachep, flags, node, size); in __do_kmalloc_node()
3615 ret = kasan_kmalloc(cachep, ret, size, flags); in __do_kmalloc_node()
3645 struct kmem_cache *cachep; in __do_kmalloc() local
3650 cachep = kmalloc_slab(size, flags); in __do_kmalloc()
3651 if (unlikely(ZERO_OR_NULL_PTR(cachep))) in __do_kmalloc()
3652 return cachep; in __do_kmalloc()
3653 ret = slab_alloc(cachep, flags, caller); in __do_kmalloc()
3655 ret = kasan_kmalloc(cachep, ret, size, flags); in __do_kmalloc()
3657 size, cachep->size, flags); in __do_kmalloc()
3682 void kmem_cache_free(struct kmem_cache *cachep, void *objp) in kmem_cache_free() argument
3685 cachep = cache_from_obj(cachep, objp); in kmem_cache_free()
3686 if (!cachep) in kmem_cache_free()
3690 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3691 if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) in kmem_cache_free()
3692 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3693 __cache_free(cachep, objp, _RET_IP_); in kmem_cache_free()
3764 static int setup_kmem_cache_nodes(struct kmem_cache *cachep, gfp_t gfp) in setup_kmem_cache_nodes() argument
3771 ret = setup_kmem_cache_node(cachep, node, gfp, true); in setup_kmem_cache_nodes()
3780 if (!cachep->list.next) { in setup_kmem_cache_nodes()
3784 n = get_node(cachep, node); in setup_kmem_cache_nodes()
3789 cachep->node[node] = NULL; in setup_kmem_cache_nodes()
3798 static int __do_tune_cpucache(struct kmem_cache *cachep, int limit, in __do_tune_cpucache() argument
3804 cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount); in __do_tune_cpucache()
3808 prev = cachep->cpu_cache; in __do_tune_cpucache()
3809 cachep->cpu_cache = cpu_cache; in __do_tune_cpucache()
3818 cachep->batchcount = batchcount; in __do_tune_cpucache()
3819 cachep->limit = limit; in __do_tune_cpucache()
3820 cachep->shared = shared; in __do_tune_cpucache()
3832 n = get_node(cachep, node); in __do_tune_cpucache()
3834 free_block(cachep, ac->entry, ac->avail, node, &list); in __do_tune_cpucache()
3836 slabs_destroy(cachep, &list); in __do_tune_cpucache()
3841 return setup_kmem_cache_nodes(cachep, gfp); in __do_tune_cpucache()
3844 static int do_tune_cpucache(struct kmem_cache *cachep, int limit, in do_tune_cpucache() argument
3850 ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp); in do_tune_cpucache()
3855 if ((ret < 0) || !is_root_cache(cachep)) in do_tune_cpucache()
3859 for_each_memcg_cache(c, cachep) { in do_tune_cpucache()
3868 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) in enable_cpucache() argument
3875 err = cache_random_seq_create(cachep, cachep->num, gfp); in enable_cpucache()
3879 if (!is_root_cache(cachep)) { in enable_cpucache()
3880 struct kmem_cache *root = memcg_root_cache(cachep); in enable_cpucache()
3897 if (cachep->size > 131072) in enable_cpucache()
3899 else if (cachep->size > PAGE_SIZE) in enable_cpucache()
3901 else if (cachep->size > 1024) in enable_cpucache()
3903 else if (cachep->size > 256) in enable_cpucache()
3918 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) in enable_cpucache()
3931 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp); in enable_cpucache()
3935 cachep->name, -err); in enable_cpucache()
3944 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, in drain_array() argument
3961 drain_array_locked(cachep, ac, node, false, &list); in drain_array()
3964 slabs_destroy(cachep, &list); in drain_array()
4036 void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) in get_slabinfo() argument
4044 for_each_kmem_cache_node(cachep, node, n) { in get_slabinfo()
4057 num_objs = total_slabs * cachep->num; in get_slabinfo()
4066 sinfo->limit = cachep->limit; in get_slabinfo()
4067 sinfo->batchcount = cachep->batchcount; in get_slabinfo()
4068 sinfo->shared = cachep->shared; in get_slabinfo()
4069 sinfo->objects_per_slab = cachep->num; in get_slabinfo()
4070 sinfo->cache_order = cachep->gfporder; in get_slabinfo()
4073 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep) in slabinfo_show_stats() argument
4077 unsigned long high = cachep->high_mark; in slabinfo_show_stats()
4078 unsigned long allocs = cachep->num_allocations; in slabinfo_show_stats()
4079 unsigned long grown = cachep->grown; in slabinfo_show_stats()
4080 unsigned long reaped = cachep->reaped; in slabinfo_show_stats()
4081 unsigned long errors = cachep->errors; in slabinfo_show_stats()
4082 unsigned long max_freeable = cachep->max_freeable; in slabinfo_show_stats()
4083 unsigned long node_allocs = cachep->node_allocs; in slabinfo_show_stats()
4084 unsigned long node_frees = cachep->node_frees; in slabinfo_show_stats()
4085 unsigned long overflows = cachep->node_overflow; in slabinfo_show_stats()
4094 unsigned long allochit = atomic_read(&cachep->allochit); in slabinfo_show_stats()
4095 unsigned long allocmiss = atomic_read(&cachep->allocmiss); in slabinfo_show_stats()
4096 unsigned long freehit = atomic_read(&cachep->freehit); in slabinfo_show_stats()
4097 unsigned long freemiss = atomic_read(&cachep->freemiss); in slabinfo_show_stats()
4120 struct kmem_cache *cachep; in slabinfo_write() local
4139 list_for_each_entry(cachep, &slab_caches, list) { in slabinfo_write()
4140 if (!strcmp(cachep->name, kbuf)) { in slabinfo_write()
4145 res = do_tune_cpucache(cachep, limit, in slabinfo_write()
4170 struct kmem_cache *cachep; in __check_heap_object() local
4177 cachep = page->slab_cache; in __check_heap_object()
4178 objnr = obj_to_index(cachep, page, (void *)ptr); in __check_heap_object()
4179 BUG_ON(objnr >= cachep->num); in __check_heap_object()
4182 offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep); in __check_heap_object()
4185 if (offset >= cachep->useroffset && in __check_heap_object()
4186 offset - cachep->useroffset <= cachep->usersize && in __check_heap_object()
4187 n <= cachep->useroffset - offset + cachep->usersize) in __check_heap_object()
4197 offset <= cachep->object_size && in __check_heap_object()
4198 n <= cachep->object_size - offset) { in __check_heap_object()
4199 usercopy_warn("SLAB object", cachep->name, to_user, offset, n); in __check_heap_object()
4203 usercopy_abort("SLAB object", cachep->name, to_user, offset, n); in __check_heap_object()