Lines Matching +full:page +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0
7 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
9 * Major cleanup, different bufctl logic, per-cpu arrays
17 * Pub: Prentice Hall ISBN 0-13-101908-2
19 * The Slab Allocator: An Object-Caching Kernel Memory Allocator
26 * page long) and always contiguous), and each slab contains multiple
48 * Each cache has a short per-cpu head array, most allocs
54 * The c_cpuarray may not be read with enabled local interrupts -
61 * The per-cpu arrays are never accessed from the wrong cpu, no locking,
62 * and local interrupts are disabled so slab code is preempt-safe.
63 * The non-constant members are protected with a per-cache irq spinlock.
65 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
66 * in 2000 - many ideas in the current implementation are derived from
71 * 11 April '97. Started multi-threading - markhe
72 * The global cache-chain is protected by the mutex 'slab_mutex'.
73 * The sem is only needed when accessing/extending the cache-chain, which
113 #include <linux/fault-inject.h>
125 #include <asm/page.h>
134 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
137 * STATS - 1 to collect stats for /proc/slabinfo.
140 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
170 #define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
176 * - LIFO ordering, to hand out cache-warm objects from _alloc
177 * - reduce the number of linked list operations
178 * - reduce spinlock operations
180 * The limit is stored in the per-cpu structure to reduce the data cache
220 struct kmem_cache_node *n, struct page *page,
228 INIT_LIST_HEAD(&parent->slabs_full); in kmem_cache_node_init()
229 INIT_LIST_HEAD(&parent->slabs_partial); in kmem_cache_node_init()
230 INIT_LIST_HEAD(&parent->slabs_free); in kmem_cache_node_init()
231 parent->total_slabs = 0; in kmem_cache_node_init()
232 parent->free_slabs = 0; in kmem_cache_node_init()
233 parent->shared = NULL; in kmem_cache_node_init()
234 parent->alien = NULL; in kmem_cache_node_init()
235 parent->colour_next = 0; in kmem_cache_node_init()
236 spin_lock_init(&parent->list_lock); in kmem_cache_node_init()
237 parent->free_objects = 0; in kmem_cache_node_init()
238 parent->free_touched = 0; in kmem_cache_node_init()
244 list_splice(&get_node(cachep, nodeid)->slab, listp); \
249 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
250 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
251 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
256 #define OBJFREELIST_SLAB(x) ((x)->flags & CFLGS_OBJFREELIST_SLAB)
257 #define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
271 #define STATS_INC_ACTIVE(x) ((x)->num_active++)
272 #define STATS_DEC_ACTIVE(x) ((x)->num_active--)
273 #define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
274 #define STATS_INC_GROWN(x) ((x)->grown++)
275 #define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
278 if ((x)->num_active > (x)->high_mark) \
279 (x)->high_mark = (x)->num_active; \
281 #define STATS_INC_ERR(x) ((x)->errors++)
282 #define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
283 #define STATS_INC_NODEFREES(x) ((x)->node_frees++)
284 #define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
287 if ((x)->max_freeable < i) \
288 (x)->max_freeable = i; \
290 #define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
291 #define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
292 #define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
293 #define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
317 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
320 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
322 * cachep->obj_offset: The real object.
323 * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
324 * cachep->size - 1* BYTES_PER_WORD: last caller address
329 return cachep->obj_offset; in obj_offset()
334 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone1()
335 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
341 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone2()
342 if (cachep->flags & SLAB_STORE_USER) in dbg_redzone2()
343 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
344 sizeof(unsigned long long) - in dbg_redzone2()
346 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
352 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); in dbg_userword()
353 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
374 static inline void *index_to_obj(struct kmem_cache *cache, struct page *page, in index_to_obj() argument
377 return page->s_mem + cache->size * idx; in index_to_obj()
386 .size = sizeof(struct kmem_cache),
394 return this_cpu_ptr(cachep->cpu_cache); in cpu_cache_get()
398 * Calculate the number of objects and left-over bytes for a given buffer size.
411 * - @buffer_size bytes for each object in cache_estimate()
412 * - One freelist_idx_t for each object in cache_estimate()
415 * freelist will be at the end of slab page. The objects will be in cache_estimate()
419 * alignment will already be calculated into the size. Because in cache_estimate()
442 function, cachep->name, msg); in __slab_error()
468 min(slab_max_order, MAX_ORDER - 1); in slab_max_order_setup()
514 if (reap_work->work.func == NULL) { in start_cpu_timer()
525 ac->avail = 0; in init_arraycache()
526 ac->limit = limit; in init_arraycache()
527 ac->batchcount = batch; in init_arraycache()
528 ac->touched = 0; in init_arraycache()
552 struct page *page, void *objp) in cache_free_pfmemalloc() argument
558 page_node = page_to_nid(page); in cache_free_pfmemalloc()
561 spin_lock(&n->list_lock); in cache_free_pfmemalloc()
563 spin_unlock(&n->list_lock); in cache_free_pfmemalloc()
578 int nr = min3(from->avail, max, to->limit - to->avail); in transfer_objects()
583 memcpy(to->entry + to->avail, from->entry + from->avail -nr, in transfer_objects()
586 from->avail -= nr; in transfer_objects()
587 to->avail += nr; in transfer_objects()
591 /* &alien->lock must be held by alien callers. */
594 /* Avoid trivial double-free. */ in __free_one()
596 WARN_ON_ONCE(ac->avail > 0 && ac->entry[ac->avail - 1] == objp)) in __free_one()
598 ac->entry[ac->avail++] = objp; in __free_one()
652 init_arraycache(&alc->ac, entries, batch); in __alloc_alien_cache()
653 spin_lock_init(&alc->lock); in __alloc_alien_cache()
674 for (i--; i >= 0; i--) in alloc_alien_cache()
700 if (ac->avail) { in __drain_alien_cache()
701 spin_lock(&n->list_lock); in __drain_alien_cache()
707 if (n->shared) in __drain_alien_cache()
708 transfer_objects(n->shared, ac, ac->limit); in __drain_alien_cache()
710 free_block(cachep, ac->entry, ac->avail, node, list); in __drain_alien_cache()
711 ac->avail = 0; in __drain_alien_cache()
712 spin_unlock(&n->list_lock); in __drain_alien_cache()
723 if (n->alien) { in reap_alien()
724 struct alien_cache *alc = n->alien[node]; in reap_alien()
728 ac = &alc->ac; in reap_alien()
729 if (ac->avail && spin_trylock_irq(&alc->lock)) { in reap_alien()
733 spin_unlock_irq(&alc->lock); in reap_alien()
753 ac = &alc->ac; in drain_alien_cache()
754 spin_lock_irqsave(&alc->lock, flags); in drain_alien_cache()
756 spin_unlock_irqrestore(&alc->lock, flags); in drain_alien_cache()
772 if (n->alien && n->alien[page_node]) { in __cache_free_alien()
773 alien = n->alien[page_node]; in __cache_free_alien()
774 ac = &alien->ac; in __cache_free_alien()
775 spin_lock(&alien->lock); in __cache_free_alien()
776 if (unlikely(ac->avail == ac->limit)) { in __cache_free_alien()
781 spin_unlock(&alien->lock); in __cache_free_alien()
785 spin_lock(&n->list_lock); in __cache_free_alien()
787 spin_unlock(&n->list_lock); in __cache_free_alien()
828 spin_lock_irq(&n->list_lock); in init_cache_node()
829 n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + in init_cache_node()
830 cachep->num; in init_cache_node()
831 spin_unlock_irq(&n->list_lock); in init_cache_node()
838 return -ENOMEM; in init_cache_node()
841 n->next_reap = jiffies + REAPTIMEOUT_NODE + in init_cache_node()
844 n->free_limit = in init_cache_node()
845 (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; in init_cache_node()
852 cachep->node[node] = n; in init_cache_node()
860 * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
861 * will be allocated off-node since memory is not yet online for the new node.
885 int ret = -ENOMEM; in setup_kmem_cache_node()
893 new_alien = alloc_alien_cache(node, cachep->limit, gfp); in setup_kmem_cache_node()
898 if (cachep->shared) { in setup_kmem_cache_node()
900 cachep->shared * cachep->batchcount, 0xbaadf00d, gfp); in setup_kmem_cache_node()
910 spin_lock_irq(&n->list_lock); in setup_kmem_cache_node()
911 if (n->shared && force_change) { in setup_kmem_cache_node()
912 free_block(cachep, n->shared->entry, in setup_kmem_cache_node()
913 n->shared->avail, node, &list); in setup_kmem_cache_node()
914 n->shared->avail = 0; in setup_kmem_cache_node()
917 if (!n->shared || force_change) { in setup_kmem_cache_node()
918 old_shared = n->shared; in setup_kmem_cache_node()
919 n->shared = new_shared; in setup_kmem_cache_node()
923 if (!n->alien) { in setup_kmem_cache_node()
924 n->alien = new_alien; in setup_kmem_cache_node()
928 spin_unlock_irq(&n->list_lock); in setup_kmem_cache_node()
932 * To protect lockless access to n->shared during irq disabled context. in setup_kmem_cache_node()
933 * If n->shared isn't NULL in irq disabled context, accessing to it is in setup_kmem_cache_node()
934 * guaranteed to be valid until irq is re-enabled, because it will be in setup_kmem_cache_node()
967 spin_lock_irq(&n->list_lock); in cpuup_canceled()
970 n->free_limit -= cachep->batchcount; in cpuup_canceled()
973 nc = per_cpu_ptr(cachep->cpu_cache, cpu); in cpuup_canceled()
974 free_block(cachep, nc->entry, nc->avail, node, &list); in cpuup_canceled()
975 nc->avail = 0; in cpuup_canceled()
978 spin_unlock_irq(&n->list_lock); in cpuup_canceled()
982 shared = n->shared; in cpuup_canceled()
984 free_block(cachep, shared->entry, in cpuup_canceled()
985 shared->avail, node, &list); in cpuup_canceled()
986 n->shared = NULL; in cpuup_canceled()
989 alien = n->alien; in cpuup_canceled()
990 n->alien = NULL; in cpuup_canceled()
992 spin_unlock_irq(&n->list_lock); in cpuup_canceled()
1045 return -ENOMEM; in cpuup_prepare()
1099 * Drains freelist for a node on each slab cache, used for memory hot-remove.
1100 * Returns -EBUSY if all objects cannot be drained so that the node is not
1119 if (!list_empty(&n->slabs_full) || in drain_cache_node_node()
1120 !list_empty(&n->slabs_partial)) { in drain_cache_node_node()
1121 ret = -EBUSY; in drain_cache_node_node()
1135 nid = mnb->status_change_nid; in slab_memory_callback()
1176 spin_lock_init(&ptr->list_lock); in init_list()
1179 cachep->node[nodeid] = ptr; in init_list()
1184 * size of kmem_cache_node.
1191 cachep->node[node] = &init_kmem_cache_node[index + node]; in set_up_node()
1192 cachep->node[node]->next_reap = jiffies + in set_up_node()
1199 * Initialisation. Called after the page allocator have been initialised and
1215 * Fragmentation resistance on low memory - only use bigger in kmem_cache_init()
1216 * page orders on machines with more than 32MB of memory if in kmem_cache_init()
1245 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids in kmem_cache_init()
1251 list_add(&kmem_cache->list, &slab_caches); in kmem_cache_init()
1260 kmalloc_info[INDEX_NODE].size, in kmem_cache_init()
1262 kmalloc_info[INDEX_NODE].size); in kmem_cache_init()
1316 * Register the timers that return unneeded pages to the page allocator in cpucache_init()
1341 pr_warn(" cache: %s, object size: %d, order: %d\n", in slab_out_of_memory()
1342 cachep->name, cachep->size, cachep->gfporder); in slab_out_of_memory()
1347 spin_lock_irqsave(&n->list_lock, flags); in slab_out_of_memory()
1348 total_slabs = n->total_slabs; in slab_out_of_memory()
1349 free_slabs = n->free_slabs; in slab_out_of_memory()
1350 free_objs = n->free_objects; in slab_out_of_memory()
1351 spin_unlock_irqrestore(&n->list_lock, flags); in slab_out_of_memory()
1354 node, total_slabs - free_slabs, total_slabs, in slab_out_of_memory()
1355 (total_slabs * cachep->num) - free_objs, in slab_out_of_memory()
1356 total_slabs * cachep->num); in slab_out_of_memory()
1362 * Interface to system's page allocator. No need to hold the
1363 * kmem_cache_node ->list_lock.
1369 static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, in kmem_getpages()
1372 struct page *page; in kmem_getpages() local
1374 flags |= cachep->allocflags; in kmem_getpages()
1376 page = __alloc_pages_node(nodeid, flags, cachep->gfporder); in kmem_getpages()
1377 if (!page) { in kmem_getpages()
1382 account_slab_page(page, cachep->gfporder, cachep); in kmem_getpages()
1383 __SetPageSlab(page); in kmem_getpages()
1385 if (sk_memalloc_socks() && page_is_pfmemalloc(page)) in kmem_getpages()
1386 SetPageSlabPfmemalloc(page); in kmem_getpages()
1388 return page; in kmem_getpages()
1392 * Interface to system's page release.
1394 static void kmem_freepages(struct kmem_cache *cachep, struct page *page) in kmem_freepages() argument
1396 int order = cachep->gfporder; in kmem_freepages()
1398 BUG_ON(!PageSlab(page)); in kmem_freepages()
1399 __ClearPageSlabPfmemalloc(page); in kmem_freepages()
1400 __ClearPageSlab(page); in kmem_freepages()
1401 page_mapcount_reset(page); in kmem_freepages()
1402 page->mapping = NULL; in kmem_freepages()
1404 if (current->reclaim_state) in kmem_freepages()
1405 current->reclaim_state->reclaimed_slab += 1 << order; in kmem_freepages()
1406 unaccount_slab_page(page, order, cachep); in kmem_freepages()
1407 __free_pages(page, order); in kmem_freepages()
1413 struct page *page; in kmem_rcu_free() local
1415 page = container_of(head, struct page, rcu_head); in kmem_rcu_free()
1416 cachep = page->slab_cache; in kmem_rcu_free()
1418 kmem_freepages(cachep, page); in kmem_rcu_free()
1425 (cachep->size % PAGE_SIZE) == 0) in is_debug_pagealloc_cache()
1437 kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map); in slab_kernel_map()
1448 int size = cachep->object_size; in poison_obj() local
1451 memset(addr, val, size); in poison_obj()
1452 *(unsigned char *)(addr + size - 1) = POISON_END; in poison_obj()
1473 if (!(error & (error - 1))) { in dump_line()
1489 int i, size; in print_objinfo() local
1492 if (cachep->flags & SLAB_RED_ZONE) { in print_objinfo()
1498 if (cachep->flags & SLAB_STORE_USER) in print_objinfo()
1501 size = cachep->object_size; in print_objinfo()
1502 for (i = 0; i < size && lines; i += 16, lines--) { in print_objinfo()
1505 if (i + limit > size) in print_objinfo()
1506 limit = size - i; in print_objinfo()
1514 int size, i; in check_poison_obj() local
1521 size = cachep->object_size; in check_poison_obj()
1523 for (i = 0; i < size; i++) { in check_poison_obj()
1525 if (i == size - 1) in check_poison_obj()
1533 print_tainted(), cachep->name, in check_poison_obj()
1534 realobj, size); in check_poison_obj()
1540 if (i + limit > size) in check_poison_obj()
1541 limit = size - i; in check_poison_obj()
1554 struct page *page = virt_to_head_page(objp); in check_poison_obj() local
1557 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1559 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1561 pr_err("Prev obj: start=%px, len=%d\n", realobj, size); in check_poison_obj()
1564 if (objnr + 1 < cachep->num) { in check_poison_obj()
1565 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1567 pr_err("Next obj: start=%px, len=%d\n", realobj, size); in check_poison_obj()
1576 struct page *page) in slab_destroy_debugcheck() argument
1580 if (OBJFREELIST_SLAB(cachep) && cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1581 poison_obj(cachep, page->freelist - obj_offset(cachep), in slab_destroy_debugcheck()
1585 for (i = 0; i < cachep->num; i++) { in slab_destroy_debugcheck()
1586 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck()
1588 if (cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1592 if (cachep->flags & SLAB_RED_ZONE) { in slab_destroy_debugcheck()
1602 struct page *page) in slab_destroy_debugcheck() argument
1608 * slab_destroy - destroy and release all objects in a slab
1610 * @page: page pointer being destroyed
1612 * Destroy all the objs in a slab page, and release the mem back to the system.
1613 * Before calling the slab page must have been unlinked from the cache. The
1614 * kmem_cache_node ->list_lock is not held/needed.
1616 static void slab_destroy(struct kmem_cache *cachep, struct page *page) in slab_destroy() argument
1620 freelist = page->freelist; in slab_destroy()
1621 slab_destroy_debugcheck(cachep, page); in slab_destroy()
1622 if (unlikely(cachep->flags & SLAB_TYPESAFE_BY_RCU)) in slab_destroy()
1623 call_rcu(&page->rcu_head, kmem_rcu_free); in slab_destroy()
1625 kmem_freepages(cachep, page); in slab_destroy()
1629 * although actual page can be freed in rcu context in slab_destroy()
1632 kmem_cache_free(cachep->freelist_cache, freelist); in slab_destroy()
1636 * Update the size of the caches before calling slabs_destroy as it may
1641 struct page *page, *n; in slabs_destroy() local
1643 list_for_each_entry_safe(page, n, list, slab_list) { in slabs_destroy()
1644 list_del(&page->slab_list); in slabs_destroy()
1645 slab_destroy(cachep, page); in slabs_destroy()
1650 * calculate_slab_order - calculate size (page order) of slabs
1652 * @size: size of objects to be created in this cache.
1659 * towards high-order requests, this should be changed.
1661 * Return: number of left-over bytes in a slab
1664 size_t size, slab_flags_t flags) in calculate_slab_order() argument
1673 num = cache_estimate(gfporder, size, flags, &remainder); in calculate_slab_order()
1698 if (freelist_cache->size > cachep->size / 2) in calculate_slab_order()
1702 /* Found something acceptable - save it away */ in calculate_slab_order()
1703 cachep->num = num; in calculate_slab_order()
1704 cachep->gfporder = gfporder; in calculate_slab_order()
1708 * A VFS-reclaimable slab tends to have most allocations in calculate_slab_order()
1710 * higher-order pages when we are unable to shrink dcache. in calculate_slab_order()
1735 size_t size; in alloc_kmem_cache_cpus() local
1738 size = sizeof(void *) * entries + sizeof(struct array_cache); in alloc_kmem_cache_cpus()
1739 cpu_cache = __alloc_percpu(size, sizeof(void *)); in alloc_kmem_cache_cpus()
1757 cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1); in setup_cpu_cache()
1758 if (!cachep->cpu_cache) in setup_cpu_cache()
1771 cachep->node[node] = kmalloc_node( in setup_cpu_cache()
1773 BUG_ON(!cachep->node[node]); in setup_cpu_cache()
1774 kmem_cache_node_init(cachep->node[node]); in setup_cpu_cache()
1778 cachep->node[numa_mem_id()]->next_reap = in setup_cpu_cache()
1782 cpu_cache_get(cachep)->avail = 0; in setup_cpu_cache()
1783 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
1784 cpu_cache_get(cachep)->batchcount = 1; in setup_cpu_cache()
1785 cpu_cache_get(cachep)->touched = 0; in setup_cpu_cache()
1786 cachep->batchcount = 1; in setup_cpu_cache()
1787 cachep->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
1798 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, in __kmem_cache_alias() argument
1803 cachep = find_mergeable(size, align, flags, name, ctor); in __kmem_cache_alias()
1805 cachep->refcount++; in __kmem_cache_alias()
1811 cachep->object_size = max_t(int, cachep->object_size, size); in __kmem_cache_alias()
1817 size_t size, slab_flags_t flags) in set_objfreelist_slab_cache() argument
1821 cachep->num = 0; in set_objfreelist_slab_cache()
1824 * If slab auto-initialization on free is enabled, store the freelist in set_objfreelist_slab_cache()
1825 * off-slab, so that its contents don't end up in one of the allocated in set_objfreelist_slab_cache()
1831 if (cachep->ctor || flags & SLAB_TYPESAFE_BY_RCU) in set_objfreelist_slab_cache()
1834 left = calculate_slab_order(cachep, size, in set_objfreelist_slab_cache()
1836 if (!cachep->num) in set_objfreelist_slab_cache()
1839 if (cachep->num * sizeof(freelist_idx_t) > cachep->object_size) in set_objfreelist_slab_cache()
1842 cachep->colour = left / cachep->colour_off; in set_objfreelist_slab_cache()
1848 size_t size, slab_flags_t flags) in set_off_slab_cache() argument
1852 cachep->num = 0; in set_off_slab_cache()
1855 * Always use on-slab management when SLAB_NOLEAKTRACE in set_off_slab_cache()
1862 * Size is large, assume best to place the slab management obj in set_off_slab_cache()
1863 * off-slab (should allow better packing of objs). in set_off_slab_cache()
1865 left = calculate_slab_order(cachep, size, flags | CFLGS_OFF_SLAB); in set_off_slab_cache()
1866 if (!cachep->num) in set_off_slab_cache()
1870 * If the slab has been placed off-slab, and we have enough space then in set_off_slab_cache()
1871 * move it on-slab. This is at the expense of any extra colouring. in set_off_slab_cache()
1873 if (left >= cachep->num * sizeof(freelist_idx_t)) in set_off_slab_cache()
1876 cachep->colour = left / cachep->colour_off; in set_off_slab_cache()
1882 size_t size, slab_flags_t flags) in set_on_slab_cache() argument
1886 cachep->num = 0; in set_on_slab_cache()
1888 left = calculate_slab_order(cachep, size, flags); in set_on_slab_cache()
1889 if (!cachep->num) in set_on_slab_cache()
1892 cachep->colour = left / cachep->colour_off; in set_on_slab_cache()
1898 * __kmem_cache_create - Create a cache.
1908 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
1911 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
1914 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
1925 unsigned int size = cachep->size; in __kmem_cache_create() local
1931 * large objects, if the increased size would increase the object size in __kmem_cache_create()
1935 if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN + in __kmem_cache_create()
1944 * Check that size is in terms of words. This is needed to avoid in __kmem_cache_create()
1946 * sure any on-slab bufctl's are also correctly aligned. in __kmem_cache_create()
1948 size = ALIGN(size, BYTES_PER_WORD); in __kmem_cache_create()
1953 * aligned, by adjusting the object size accordingly. */ in __kmem_cache_create()
1954 size = ALIGN(size, REDZONE_ALIGN); in __kmem_cache_create()
1958 if (ralign < cachep->align) { in __kmem_cache_create()
1959 ralign = cachep->align; in __kmem_cache_create()
1967 cachep->align = ralign; in __kmem_cache_create()
1968 cachep->colour_off = cache_line_size(); in __kmem_cache_create()
1970 if (cachep->colour_off < cachep->align) in __kmem_cache_create()
1971 cachep->colour_off = cachep->align; in __kmem_cache_create()
1981 * Both debugging options require word-alignment which is calculated in __kmem_cache_create()
1986 cachep->obj_offset += sizeof(unsigned long long); in __kmem_cache_create()
1987 size += 2 * sizeof(unsigned long long); in __kmem_cache_create()
1995 size += REDZONE_ALIGN; in __kmem_cache_create()
1997 size += BYTES_PER_WORD; in __kmem_cache_create()
2001 kasan_cache_create(cachep, &size, &flags); in __kmem_cache_create()
2003 size = ALIGN(size, cachep->align); in __kmem_cache_create()
2008 if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE) in __kmem_cache_create()
2009 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align); in __kmem_cache_create()
2013 * To activate debug pagealloc, off-slab management is necessary in __kmem_cache_create()
2016 * to check size >= 256. It guarantees that all necessary small in __kmem_cache_create()
2020 size >= 256 && cachep->object_size > cache_line_size()) { in __kmem_cache_create()
2021 if (size < PAGE_SIZE || size % PAGE_SIZE == 0) { in __kmem_cache_create()
2022 size_t tmp_size = ALIGN(size, PAGE_SIZE); in __kmem_cache_create()
2026 cachep->obj_offset += tmp_size - size; in __kmem_cache_create()
2027 size = tmp_size; in __kmem_cache_create()
2034 if (set_objfreelist_slab_cache(cachep, size, flags)) { in __kmem_cache_create()
2039 if (set_off_slab_cache(cachep, size, flags)) { in __kmem_cache_create()
2044 if (set_on_slab_cache(cachep, size, flags)) in __kmem_cache_create()
2047 return -E2BIG; in __kmem_cache_create()
2050 cachep->freelist_size = cachep->num * sizeof(freelist_idx_t); in __kmem_cache_create()
2051 cachep->flags = flags; in __kmem_cache_create()
2052 cachep->allocflags = __GFP_COMP; in __kmem_cache_create()
2054 cachep->allocflags |= GFP_DMA; in __kmem_cache_create()
2056 cachep->allocflags |= GFP_DMA32; in __kmem_cache_create()
2058 cachep->allocflags |= __GFP_RECLAIMABLE; in __kmem_cache_create()
2059 cachep->size = size; in __kmem_cache_create()
2060 cachep->reciprocal_buffer_size = reciprocal_value(size); in __kmem_cache_create()
2069 (cachep->flags & SLAB_POISON) && in __kmem_cache_create()
2071 cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); in __kmem_cache_create()
2075 cachep->freelist_cache = in __kmem_cache_create()
2076 kmalloc_slab(cachep->freelist_size, 0u); in __kmem_cache_create()
2108 assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock); in check_spinlock_acquired()
2116 assert_spin_locked(&get_node(cachep, node)->list_lock); in check_spinlock_acquired_node()
2133 if (!ac || !ac->avail) in drain_array_locked()
2136 tofree = free_all ? ac->avail : (ac->limit + 4) / 5; in drain_array_locked()
2137 if (tofree > ac->avail) in drain_array_locked()
2138 tofree = (ac->avail + 1) / 2; in drain_array_locked()
2140 free_block(cachep, ac->entry, tofree, node, list); in drain_array_locked()
2141 ac->avail -= tofree; in drain_array_locked()
2142 memmove(ac->entry, &(ac->entry[tofree]), sizeof(void *) * ac->avail); in drain_array_locked()
2156 spin_lock(&n->list_lock); in do_drain()
2157 free_block(cachep, ac->entry, ac->avail, node, &list); in do_drain()
2158 spin_unlock(&n->list_lock); in do_drain()
2159 ac->avail = 0; in do_drain()
2172 if (n->alien) in drain_cpu_caches()
2173 drain_alien_cache(cachep, n->alien); in drain_cpu_caches()
2176 spin_lock_irq(&n->list_lock); in drain_cpu_caches()
2177 drain_array_locked(cachep, n->shared, node, true, &list); in drain_cpu_caches()
2178 spin_unlock_irq(&n->list_lock); in drain_cpu_caches()
2195 struct page *page; in drain_freelist() local
2198 while (nr_freed < tofree && !list_empty(&n->slabs_free)) { in drain_freelist()
2200 spin_lock_irq(&n->list_lock); in drain_freelist()
2201 p = n->slabs_free.prev; in drain_freelist()
2202 if (p == &n->slabs_free) { in drain_freelist()
2203 spin_unlock_irq(&n->list_lock); in drain_freelist()
2207 page = list_entry(p, struct page, slab_list); in drain_freelist()
2208 list_del(&page->slab_list); in drain_freelist()
2209 n->free_slabs--; in drain_freelist()
2210 n->total_slabs--; in drain_freelist()
2215 n->free_objects -= cache->num; in drain_freelist()
2216 spin_unlock_irq(&n->list_lock); in drain_freelist()
2217 slab_destroy(cache, page); in drain_freelist()
2230 if (!list_empty(&n->slabs_full) || in __kmem_cache_empty()
2231 !list_empty(&n->slabs_partial)) in __kmem_cache_empty()
2248 ret += !list_empty(&n->slabs_full) || in __kmem_cache_shrink()
2249 !list_empty(&n->slabs_partial); in __kmem_cache_shrink()
2266 free_percpu(cachep->cpu_cache); in __kmem_cache_release()
2270 kfree(n->shared); in __kmem_cache_release()
2271 free_alien_cache(n->alien); in __kmem_cache_release()
2273 cachep->node[i] = NULL; in __kmem_cache_release()
2280 * For a slab cache when the slab descriptor is off-slab, the
2283 * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
2285 * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
2286 * This is a "chicken-and-egg" problem.
2288 * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
2292 struct page *page, int colour_off, in alloc_slabmgmt() argument
2296 void *addr = page_address(page); in alloc_slabmgmt()
2298 page->s_mem = addr + colour_off; in alloc_slabmgmt()
2299 page->active = 0; in alloc_slabmgmt()
2304 /* Slab management obj is off-slab. */ in alloc_slabmgmt()
2305 freelist = kmem_cache_alloc_node(cachep->freelist_cache, in alloc_slabmgmt()
2309 freelist = addr + (PAGE_SIZE << cachep->gfporder) - in alloc_slabmgmt()
2310 cachep->freelist_size; in alloc_slabmgmt()
2316 static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx) in get_free_obj() argument
2318 return ((freelist_idx_t *)page->freelist)[idx]; in get_free_obj()
2321 static inline void set_free_obj(struct page *page, in set_free_obj() argument
2324 ((freelist_idx_t *)(page->freelist))[idx] = val; in set_free_obj()
2327 static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page) in cache_init_objs_debug() argument
2332 for (i = 0; i < cachep->num; i++) { in cache_init_objs_debug()
2333 void *objp = index_to_obj(cachep, page, i); in cache_init_objs_debug()
2335 if (cachep->flags & SLAB_STORE_USER) in cache_init_objs_debug()
2338 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs_debug()
2347 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) { in cache_init_objs_debug()
2350 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs_debug()
2355 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs_debug()
2362 if (cachep->flags & SLAB_POISON) { in cache_init_objs_debug()
2383 * return true if the pre-computed list is available, false otherwize.
2395 /* Use a random state if the pre-computed list is not available */ in freelist_state_initialize()
2396 if (!cachep->random_seq) { in freelist_state_initialize()
2397 prandom_seed_state(&state->rnd_state, rand); in freelist_state_initialize()
2400 state->list = cachep->random_seq; in freelist_state_initialize()
2401 state->count = count; in freelist_state_initialize()
2402 state->pos = rand % count; in freelist_state_initialize()
2411 if (state->pos >= state->count) in next_random_slot()
2412 state->pos = 0; in next_random_slot()
2413 return state->list[state->pos++]; in next_random_slot()
2417 static void swap_free_obj(struct page *page, unsigned int a, unsigned int b) in swap_free_obj() argument
2419 swap(((freelist_idx_t *)page->freelist)[a], in swap_free_obj()
2420 ((freelist_idx_t *)page->freelist)[b]); in swap_free_obj()
2424 * Shuffle the freelist initialization state based on pre-computed lists.
2427 static bool shuffle_freelist(struct kmem_cache *cachep, struct page *page) in shuffle_freelist() argument
2429 unsigned int objfreelist = 0, i, rand, count = cachep->num; in shuffle_freelist()
2441 objfreelist = count - 1; in shuffle_freelist()
2444 page->freelist = index_to_obj(cachep, page, objfreelist) + in shuffle_freelist()
2446 count--; in shuffle_freelist()
2451 * Later use a pre-computed list for speed. in shuffle_freelist()
2455 set_free_obj(page, i, i); in shuffle_freelist()
2457 /* Fisher-Yates shuffle */ in shuffle_freelist()
2458 for (i = count - 1; i > 0; i--) { in shuffle_freelist()
2461 swap_free_obj(page, i, rand); in shuffle_freelist()
2465 set_free_obj(page, i, next_random_slot(&state)); in shuffle_freelist()
2469 set_free_obj(page, cachep->num - 1, objfreelist); in shuffle_freelist()
2475 struct page *page) in shuffle_freelist() argument
2482 struct page *page) in cache_init_objs() argument
2488 cache_init_objs_debug(cachep, page); in cache_init_objs()
2491 shuffled = shuffle_freelist(cachep, page); in cache_init_objs()
2494 page->freelist = index_to_obj(cachep, page, cachep->num - 1) + in cache_init_objs()
2498 for (i = 0; i < cachep->num; i++) { in cache_init_objs()
2499 objp = index_to_obj(cachep, page, i); in cache_init_objs()
2503 if (DEBUG == 0 && cachep->ctor) { in cache_init_objs()
2505 cachep->ctor(objp); in cache_init_objs()
2510 set_free_obj(page, i, i); in cache_init_objs()
2514 static void *slab_get_obj(struct kmem_cache *cachep, struct page *page) in slab_get_obj() argument
2518 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2519 page->active++; in slab_get_obj()
2525 struct page *page, void *objp) in slab_put_obj() argument
2527 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2532 for (i = page->active; i < cachep->num; i++) { in slab_put_obj()
2533 if (get_free_obj(page, i) == objnr) { in slab_put_obj()
2535 cachep->name, objp); in slab_put_obj()
2540 page->active--; in slab_put_obj()
2541 if (!page->freelist) in slab_put_obj()
2542 page->freelist = objp + obj_offset(cachep); in slab_put_obj()
2544 set_free_obj(page, page->active, objnr); in slab_put_obj()
2552 static void slab_map_pages(struct kmem_cache *cache, struct page *page, in slab_map_pages() argument
2555 page->slab_cache = cache; in slab_map_pages()
2556 page->freelist = freelist; in slab_map_pages()
2563 static struct page *cache_grow_begin(struct kmem_cache *cachep, in cache_grow_begin()
2571 struct page *page; in cache_grow_begin() local
2580 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO)); in cache_grow_begin()
2588 * Get mem for the objs. Attempt to allocate a physical page from in cache_grow_begin()
2591 page = kmem_getpages(cachep, local_flags, nodeid); in cache_grow_begin()
2592 if (!page) in cache_grow_begin()
2595 page_node = page_to_nid(page); in cache_grow_begin()
2599 n->colour_next++; in cache_grow_begin()
2600 if (n->colour_next >= cachep->colour) in cache_grow_begin()
2601 n->colour_next = 0; in cache_grow_begin()
2603 offset = n->colour_next; in cache_grow_begin()
2604 if (offset >= cachep->colour) in cache_grow_begin()
2607 offset *= cachep->colour_off; in cache_grow_begin()
2611 * page_address() in the latter returns a non-tagged pointer, in cache_grow_begin()
2614 kasan_poison_slab(page); in cache_grow_begin()
2617 freelist = alloc_slabmgmt(cachep, page, offset, in cache_grow_begin()
2622 slab_map_pages(cachep, page, freelist); in cache_grow_begin()
2624 cache_init_objs(cachep, page); in cache_grow_begin()
2629 return page; in cache_grow_begin()
2632 kmem_freepages(cachep, page); in cache_grow_begin()
2639 static void cache_grow_end(struct kmem_cache *cachep, struct page *page) in cache_grow_end() argument
2646 if (!page) in cache_grow_end()
2649 INIT_LIST_HEAD(&page->slab_list); in cache_grow_end()
2650 n = get_node(cachep, page_to_nid(page)); in cache_grow_end()
2652 spin_lock(&n->list_lock); in cache_grow_end()
2653 n->total_slabs++; in cache_grow_end()
2654 if (!page->active) { in cache_grow_end()
2655 list_add_tail(&page->slab_list, &n->slabs_free); in cache_grow_end()
2656 n->free_slabs++; in cache_grow_end()
2658 fixup_slab_list(cachep, n, page, &list); in cache_grow_end()
2661 n->free_objects += cachep->num - page->active; in cache_grow_end()
2662 spin_unlock(&n->list_lock); in cache_grow_end()
2671 * - detect bad pointers.
2672 * - POISON/RED_ZONE checking
2709 struct page *page; in cache_free_debugcheck() local
2713 objp -= obj_offset(cachep); in cache_free_debugcheck()
2715 page = virt_to_head_page(objp); in cache_free_debugcheck()
2717 if (cachep->flags & SLAB_RED_ZONE) { in cache_free_debugcheck()
2722 if (cachep->flags & SLAB_STORE_USER) in cache_free_debugcheck()
2725 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2727 BUG_ON(objnr >= cachep->num); in cache_free_debugcheck()
2728 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2730 if (cachep->flags & SLAB_POISON) { in cache_free_debugcheck()
2750 objp = next - obj_offset(cachep); in fixup_objfreelist_debug()
2758 struct kmem_cache_node *n, struct page *page, in fixup_slab_list() argument
2762 list_del(&page->slab_list); in fixup_slab_list()
2763 if (page->active == cachep->num) { in fixup_slab_list()
2764 list_add(&page->slab_list, &n->slabs_full); in fixup_slab_list()
2768 if (cachep->flags & SLAB_POISON) { in fixup_slab_list()
2769 void **objp = page->freelist; in fixup_slab_list()
2775 page->freelist = NULL; in fixup_slab_list()
2778 list_add(&page->slab_list, &n->slabs_partial); in fixup_slab_list()
2781 /* Try to find non-pfmemalloc slab if needed */
2782 static noinline struct page *get_valid_first_slab(struct kmem_cache_node *n, in get_valid_first_slab()
2783 struct page *page, bool pfmemalloc) in get_valid_first_slab() argument
2785 if (!page) in get_valid_first_slab()
2789 return page; in get_valid_first_slab()
2791 if (!PageSlabPfmemalloc(page)) in get_valid_first_slab()
2792 return page; in get_valid_first_slab()
2795 if (n->free_objects > n->free_limit) { in get_valid_first_slab()
2796 ClearPageSlabPfmemalloc(page); in get_valid_first_slab()
2797 return page; in get_valid_first_slab()
2801 list_del(&page->slab_list); in get_valid_first_slab()
2802 if (!page->active) { in get_valid_first_slab()
2803 list_add_tail(&page->slab_list, &n->slabs_free); in get_valid_first_slab()
2804 n->free_slabs++; in get_valid_first_slab()
2806 list_add_tail(&page->slab_list, &n->slabs_partial); in get_valid_first_slab()
2808 list_for_each_entry(page, &n->slabs_partial, slab_list) { in get_valid_first_slab()
2809 if (!PageSlabPfmemalloc(page)) in get_valid_first_slab()
2810 return page; in get_valid_first_slab()
2813 n->free_touched = 1; in get_valid_first_slab()
2814 list_for_each_entry(page, &n->slabs_free, slab_list) { in get_valid_first_slab()
2815 if (!PageSlabPfmemalloc(page)) { in get_valid_first_slab()
2816 n->free_slabs--; in get_valid_first_slab()
2817 return page; in get_valid_first_slab()
2824 static struct page *get_first_slab(struct kmem_cache_node *n, bool pfmemalloc) in get_first_slab()
2826 struct page *page; in get_first_slab() local
2828 assert_spin_locked(&n->list_lock); in get_first_slab()
2829 page = list_first_entry_or_null(&n->slabs_partial, struct page, in get_first_slab()
2831 if (!page) { in get_first_slab()
2832 n->free_touched = 1; in get_first_slab()
2833 page = list_first_entry_or_null(&n->slabs_free, struct page, in get_first_slab()
2835 if (page) in get_first_slab()
2836 n->free_slabs--; in get_first_slab()
2840 page = get_valid_first_slab(n, page, pfmemalloc); in get_first_slab()
2842 return page; in get_first_slab()
2848 struct page *page; in cache_alloc_pfmemalloc() local
2855 spin_lock(&n->list_lock); in cache_alloc_pfmemalloc()
2856 page = get_first_slab(n, true); in cache_alloc_pfmemalloc()
2857 if (!page) { in cache_alloc_pfmemalloc()
2858 spin_unlock(&n->list_lock); in cache_alloc_pfmemalloc()
2862 obj = slab_get_obj(cachep, page); in cache_alloc_pfmemalloc()
2863 n->free_objects--; in cache_alloc_pfmemalloc()
2865 fixup_slab_list(cachep, n, page, &list); in cache_alloc_pfmemalloc()
2867 spin_unlock(&n->list_lock); in cache_alloc_pfmemalloc()
2878 struct array_cache *ac, struct page *page, int batchcount) in alloc_block() argument
2884 BUG_ON(page->active >= cachep->num); in alloc_block()
2886 while (page->active < cachep->num && batchcount--) { in alloc_block()
2891 ac->entry[ac->avail++] = slab_get_obj(cachep, page); in alloc_block()
2904 struct page *page; in cache_alloc_refill() local
2910 batchcount = ac->batchcount; in cache_alloc_refill()
2911 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { in cache_alloc_refill()
2921 BUG_ON(ac->avail > 0 || !n); in cache_alloc_refill()
2922 shared = READ_ONCE(n->shared); in cache_alloc_refill()
2923 if (!n->free_objects && (!shared || !shared->avail)) in cache_alloc_refill()
2926 spin_lock(&n->list_lock); in cache_alloc_refill()
2927 shared = READ_ONCE(n->shared); in cache_alloc_refill()
2931 shared->touched = 1; in cache_alloc_refill()
2937 page = get_first_slab(n, false); in cache_alloc_refill()
2938 if (!page) in cache_alloc_refill()
2943 batchcount = alloc_block(cachep, ac, page, batchcount); in cache_alloc_refill()
2944 fixup_slab_list(cachep, n, page, &list); in cache_alloc_refill()
2948 n->free_objects -= ac->avail; in cache_alloc_refill()
2950 spin_unlock(&n->list_lock); in cache_alloc_refill()
2954 if (unlikely(!ac->avail)) { in cache_alloc_refill()
2963 page = cache_grow_begin(cachep, gfp_exact_node(flags), node); in cache_alloc_refill()
2970 if (!ac->avail && page) in cache_alloc_refill()
2971 alloc_block(cachep, ac, page, batchcount); in cache_alloc_refill()
2972 cache_grow_end(cachep, page); in cache_alloc_refill()
2974 if (!ac->avail) in cache_alloc_refill()
2977 ac->touched = 1; in cache_alloc_refill()
2979 return ac->entry[--ac->avail]; in cache_alloc_refill()
2992 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO)); in cache_alloc_debugcheck_after()
2995 if (cachep->flags & SLAB_POISON) { in cache_alloc_debugcheck_after()
3000 if (cachep->flags & SLAB_STORE_USER) in cache_alloc_debugcheck_after()
3003 if (cachep->flags & SLAB_RED_ZONE) { in cache_alloc_debugcheck_after()
3016 if (cachep->ctor && cachep->flags & SLAB_POISON) in cache_alloc_debugcheck_after()
3017 cachep->ctor(objp); in cache_alloc_debugcheck_after()
3019 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { in cache_alloc_debugcheck_after()
3037 if (likely(ac->avail)) { in ____cache_alloc()
3038 ac->touched = 1; in ____cache_alloc()
3039 objp = ac->entry[--ac->avail]; in ____cache_alloc()
3056 * per-CPU caches is leaked, we need to make sure kmemleak doesn't in ____cache_alloc()
3060 kmemleak_erase(&ac->entry[ac->avail]); in ____cache_alloc()
3078 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) in alternate_node_alloc()
3080 else if (current->mempolicy) in alternate_node_alloc()
3091 * perform an allocation without specifying a node. This allows the page
3102 struct page *page; in fallback_alloc() local
3123 get_node(cache, nid)->free_objects) { in fallback_alloc()
3138 page = cache_grow_begin(cache, flags, numa_mem_id()); in fallback_alloc()
3139 cache_grow_end(cache, page); in fallback_alloc()
3140 if (page) { in fallback_alloc()
3141 nid = page_to_nid(page); in fallback_alloc()
3165 struct page *page; in ____cache_alloc_node() local
3175 spin_lock(&n->list_lock); in ____cache_alloc_node()
3176 page = get_first_slab(n, false); in ____cache_alloc_node()
3177 if (!page) in ____cache_alloc_node()
3186 BUG_ON(page->active == cachep->num); in ____cache_alloc_node()
3188 obj = slab_get_obj(cachep, page); in ____cache_alloc_node()
3189 n->free_objects--; in ____cache_alloc_node()
3191 fixup_slab_list(cachep, n, page, &list); in ____cache_alloc_node()
3193 spin_unlock(&n->list_lock); in ____cache_alloc_node()
3198 spin_unlock(&n->list_lock); in ____cache_alloc_node()
3199 page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid); in ____cache_alloc_node()
3200 if (page) { in ____cache_alloc_node()
3202 obj = slab_get_obj(cachep, page); in ____cache_alloc_node()
3204 cache_grow_end(cachep, page); in ____cache_alloc_node()
3253 memset(ptr, 0, cachep->object_size); in slab_alloc_node()
3264 if (current->mempolicy || cpuset_do_slab_mem_spread()) { in __do_cache_alloc()
3311 memset(objp, 0, cachep->object_size); in slab_alloc()
3326 struct page *page; in free_block() local
3328 n->free_objects += nr_objects; in free_block()
3332 struct page *page; in free_block() local
3336 page = virt_to_head_page(objp); in free_block()
3337 list_del(&page->slab_list); in free_block()
3339 slab_put_obj(cachep, page, objp); in free_block()
3343 if (page->active == 0) { in free_block()
3344 list_add(&page->slab_list, &n->slabs_free); in free_block()
3345 n->free_slabs++; in free_block()
3348 * partial list on free - maximum time for the in free_block()
3351 list_add_tail(&page->slab_list, &n->slabs_partial); in free_block()
3355 while (n->free_objects > n->free_limit && !list_empty(&n->slabs_free)) { in free_block()
3356 n->free_objects -= cachep->num; in free_block()
3358 page = list_last_entry(&n->slabs_free, struct page, slab_list); in free_block()
3359 list_move(&page->slab_list, list); in free_block()
3360 n->free_slabs--; in free_block()
3361 n->total_slabs--; in free_block()
3372 batchcount = ac->batchcount; in cache_flusharray()
3376 spin_lock(&n->list_lock); in cache_flusharray()
3377 if (n->shared) { in cache_flusharray()
3378 struct array_cache *shared_array = n->shared; in cache_flusharray()
3379 int max = shared_array->limit - shared_array->avail; in cache_flusharray()
3383 memcpy(&(shared_array->entry[shared_array->avail]), in cache_flusharray()
3384 ac->entry, sizeof(void *) * batchcount); in cache_flusharray()
3385 shared_array->avail += batchcount; in cache_flusharray()
3390 free_block(cachep, ac->entry, batchcount, node, &list); in cache_flusharray()
3395 struct page *page; in cache_flusharray() local
3397 list_for_each_entry(page, &n->slabs_free, slab_list) { in cache_flusharray()
3398 BUG_ON(page->active); in cache_flusharray()
3405 spin_unlock(&n->list_lock); in cache_flusharray()
3406 ac->avail -= batchcount; in cache_flusharray()
3407 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); in cache_flusharray()
3422 /* Use KCSAN to help debug racy use-after-free. */ in __cache_free()
3423 if (!(cachep->flags & SLAB_TYPESAFE_BY_RCU)) in __cache_free()
3424 __kcsan_check_access(objp, cachep->object_size, in __cache_free()
3437 memset(objp, 0, cachep->object_size); in ___cache_free()
3438 kmemleak_free_recursive(objp, cachep->flags); in ___cache_free()
3445 * is per page memory reference) to get nodeid. Instead use a global in ___cache_free()
3452 if (ac->avail < ac->limit) { in ___cache_free()
3460 struct page *page = virt_to_head_page(objp); in ___cache_free() local
3462 if (unlikely(PageSlabPfmemalloc(page))) { in ___cache_free()
3463 cache_free_pfmemalloc(cachep, page, objp); in ___cache_free()
3472 * kmem_cache_alloc - Allocate an object
3486 cachep->object_size, cachep->size, flags); in kmem_cache_alloc()
3494 size_t size, void **p, unsigned long caller) in cache_alloc_debugcheck_after_bulk() argument
3498 for (i = 0; i < size; i++) in cache_alloc_debugcheck_after_bulk()
3502 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, in kmem_cache_alloc_bulk() argument
3508 s = slab_pre_alloc_hook(s, &objcg, size, flags); in kmem_cache_alloc_bulk()
3515 for (i = 0; i < size; i++) { in kmem_cache_alloc_bulk()
3524 cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_); in kmem_cache_alloc_bulk()
3528 for (i = 0; i < size; i++) in kmem_cache_alloc_bulk()
3529 memset(p[i], 0, s->object_size); in kmem_cache_alloc_bulk()
3531 slab_post_alloc_hook(s, objcg, flags, size, p); in kmem_cache_alloc_bulk()
3533 return size; in kmem_cache_alloc_bulk()
3545 kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) in kmem_cache_alloc_trace() argument
3551 ret = kasan_kmalloc(cachep, ret, size, flags); in kmem_cache_alloc_trace()
3553 size, cachep->size, flags); in kmem_cache_alloc_trace()
3561 * kmem_cache_alloc_node - Allocate an object on the specified node
3578 cachep->object_size, cachep->size, in kmem_cache_alloc_node()
3589 size_t size) in kmem_cache_alloc_node_trace() argument
3595 ret = kasan_kmalloc(cachep, ret, size, flags); in kmem_cache_alloc_node_trace()
3597 size, cachep->size, in kmem_cache_alloc_node_trace()
3605 __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) in __do_kmalloc_node() argument
3610 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) in __do_kmalloc_node()
3612 cachep = kmalloc_slab(size, flags); in __do_kmalloc_node()
3615 ret = kmem_cache_alloc_node_trace(cachep, flags, node, size); in __do_kmalloc_node()
3616 ret = kasan_kmalloc(cachep, ret, size, flags); in __do_kmalloc_node()
3621 void *__kmalloc_node(size_t size, gfp_t flags, int node) in __kmalloc_node() argument
3623 return __do_kmalloc_node(size, flags, node, _RET_IP_); in __kmalloc_node()
3627 void *__kmalloc_node_track_caller(size_t size, gfp_t flags, in __kmalloc_node_track_caller() argument
3630 return __do_kmalloc_node(size, flags, node, caller); in __kmalloc_node_track_caller()
3636 * __do_kmalloc - allocate memory
3637 * @size: how many bytes of memory are required.
3643 static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, in __do_kmalloc() argument
3649 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) in __do_kmalloc()
3651 cachep = kmalloc_slab(size, flags); in __do_kmalloc()
3656 ret = kasan_kmalloc(cachep, ret, size, flags); in __do_kmalloc()
3658 size, cachep->size, flags); in __do_kmalloc()
3663 void *__kmalloc(size_t size, gfp_t flags) in __kmalloc() argument
3665 return __do_kmalloc(size, flags, _RET_IP_); in __kmalloc()
3669 void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller) in __kmalloc_track_caller() argument
3671 return __do_kmalloc(size, flags, caller); in __kmalloc_track_caller()
3676 * kmem_cache_free - Deallocate an object
3691 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3692 if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) in kmem_cache_free()
3693 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3701 void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) in kmem_cache_free_bulk() argument
3707 for (i = 0; i < size; i++) { in kmem_cache_free_bulk()
3717 debug_check_no_locks_freed(objp, s->object_size); in kmem_cache_free_bulk()
3718 if (!(s->flags & SLAB_DEBUG_OBJECTS)) in kmem_cache_free_bulk()
3719 debug_check_no_obj_freed(objp, s->object_size); in kmem_cache_free_bulk()
3730 * kfree - free previously allocated memory
3754 debug_check_no_locks_freed(objp, c->object_size); in kfree()
3756 debug_check_no_obj_freed(objp, c->object_size); in kfree()
3781 if (!cachep->list.next) { in setup_kmem_cache_nodes()
3783 node--; in setup_kmem_cache_nodes()
3787 kfree(n->shared); in setup_kmem_cache_nodes()
3788 free_alien_cache(n->alien); in setup_kmem_cache_nodes()
3790 cachep->node[node] = NULL; in setup_kmem_cache_nodes()
3792 node--; in setup_kmem_cache_nodes()
3795 return -ENOMEM; in setup_kmem_cache_nodes()
3807 return -ENOMEM; in do_tune_cpucache()
3809 prev = cachep->cpu_cache; in do_tune_cpucache()
3810 cachep->cpu_cache = cpu_cache; in do_tune_cpucache()
3819 cachep->batchcount = batchcount; in do_tune_cpucache()
3820 cachep->limit = limit; in do_tune_cpucache()
3821 cachep->shared = shared; in do_tune_cpucache()
3834 spin_lock_irq(&n->list_lock); in do_tune_cpucache()
3835 free_block(cachep, ac->entry, ac->avail, node, &list); in do_tune_cpucache()
3836 spin_unlock_irq(&n->list_lock); in do_tune_cpucache()
3853 err = cache_random_seq_create(cachep, cachep->num, gfp); in enable_cpucache()
3861 * - create a LIFO ordering, i.e. return objects that are cache-warm in enable_cpucache()
3862 * - reduce the number of spinlock operations. in enable_cpucache()
3863 * - reduce the number of linked list operations on the slab and in enable_cpucache()
3865 * The numbers are guessed, we should auto-tune as described by in enable_cpucache()
3868 if (cachep->size > 131072) in enable_cpucache()
3870 else if (cachep->size > PAGE_SIZE) in enable_cpucache()
3872 else if (cachep->size > 1024) in enable_cpucache()
3874 else if (cachep->size > 256) in enable_cpucache()
3889 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) in enable_cpucache()
3906 cachep->name, -err); in enable_cpucache()
3920 /* ac from n->shared can be freed if we don't hold the slab_mutex. */ in drain_array()
3923 if (!ac || !ac->avail) in drain_array()
3926 if (ac->touched) { in drain_array()
3927 ac->touched = 0; in drain_array()
3931 spin_lock_irq(&n->list_lock); in drain_array()
3933 spin_unlock_irq(&n->list_lock); in drain_array()
3939 * cache_reap - Reclaim memory from caches.
3944 * - clear the per-cpu caches for this CPU.
3945 * - return freeable pages to the main free memory pool.
3947 * If we cannot acquire the cache chain mutex then just give up - we'll try
3979 if (time_after(n->next_reap, jiffies)) in cache_reap()
3982 n->next_reap = jiffies + REAPTIMEOUT_NODE; in cache_reap()
3984 drain_array(searchp, n, n->shared, node); in cache_reap()
3986 if (n->free_touched) in cache_reap()
3987 n->free_touched = 0; in cache_reap()
3991 freed = drain_freelist(searchp, n, (n->free_limit + in cache_reap()
3992 5 * searchp->num - 1) / (5 * searchp->num)); in cache_reap()
4017 spin_lock_irq(&n->list_lock); in get_slabinfo()
4019 total_slabs += n->total_slabs; in get_slabinfo()
4020 free_slabs += n->free_slabs; in get_slabinfo()
4021 free_objs += n->free_objects; in get_slabinfo()
4023 if (n->shared) in get_slabinfo()
4024 shared_avail += n->shared->avail; in get_slabinfo()
4026 spin_unlock_irq(&n->list_lock); in get_slabinfo()
4028 num_objs = total_slabs * cachep->num; in get_slabinfo()
4029 active_slabs = total_slabs - free_slabs; in get_slabinfo()
4030 active_objs = num_objs - free_objs; in get_slabinfo()
4032 sinfo->active_objs = active_objs; in get_slabinfo()
4033 sinfo->num_objs = num_objs; in get_slabinfo()
4034 sinfo->active_slabs = active_slabs; in get_slabinfo()
4035 sinfo->num_slabs = total_slabs; in get_slabinfo()
4036 sinfo->shared_avail = shared_avail; in get_slabinfo()
4037 sinfo->limit = cachep->limit; in get_slabinfo()
4038 sinfo->batchcount = cachep->batchcount; in get_slabinfo()
4039 sinfo->shared = cachep->shared; in get_slabinfo()
4040 sinfo->objects_per_slab = cachep->num; in get_slabinfo()
4041 sinfo->cache_order = cachep->gfporder; in get_slabinfo()
4048 unsigned long high = cachep->high_mark; in slabinfo_show_stats()
4049 unsigned long allocs = cachep->num_allocations; in slabinfo_show_stats()
4050 unsigned long grown = cachep->grown; in slabinfo_show_stats()
4051 unsigned long reaped = cachep->reaped; in slabinfo_show_stats()
4052 unsigned long errors = cachep->errors; in slabinfo_show_stats()
4053 unsigned long max_freeable = cachep->max_freeable; in slabinfo_show_stats()
4054 unsigned long node_allocs = cachep->node_allocs; in slabinfo_show_stats()
4055 unsigned long node_frees = cachep->node_frees; in slabinfo_show_stats()
4056 unsigned long overflows = cachep->node_overflow; in slabinfo_show_stats()
4065 unsigned long allochit = atomic_read(&cachep->allochit); in slabinfo_show_stats()
4066 unsigned long allocmiss = atomic_read(&cachep->allocmiss); in slabinfo_show_stats()
4067 unsigned long freehit = atomic_read(&cachep->freehit); in slabinfo_show_stats()
4068 unsigned long freemiss = atomic_read(&cachep->freemiss); in slabinfo_show_stats()
4078 * slabinfo_write - Tuning for the slab allocator
4094 return -EINVAL; in slabinfo_write()
4096 return -EFAULT; in slabinfo_write()
4101 return -EINVAL; in slabinfo_write()
4105 return -EINVAL; in slabinfo_write()
4109 res = -EINVAL; in slabinfo_write()
4111 if (!strcmp(cachep->name, kbuf)) { in slabinfo_write()
4138 void __check_heap_object(const void *ptr, unsigned long n, struct page *page, in __check_heap_object() argument
4148 cachep = page->slab_cache; in __check_heap_object()
4149 objnr = obj_to_index(cachep, page, (void *)ptr); in __check_heap_object()
4150 BUG_ON(objnr >= cachep->num); in __check_heap_object()
4153 offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep); in __check_heap_object()
4156 if (offset >= cachep->useroffset && in __check_heap_object()
4157 offset - cachep->useroffset <= cachep->usersize && in __check_heap_object()
4158 n <= cachep->useroffset - offset + cachep->usersize) in __check_heap_object()
4168 offset <= cachep->object_size && in __check_heap_object()
4169 n <= cachep->object_size - offset) { in __check_heap_object()
4170 usercopy_warn("SLAB object", cachep->name, to_user, offset, n); in __check_heap_object()
4174 usercopy_abort("SLAB object", cachep->name, to_user, offset, n); in __check_heap_object()
4179 * __ksize -- Uninstrumented ksize.
4185 * Return: size of the actual memory used by @objp in bytes
4190 size_t size; in __ksize() local
4197 size = c ? c->object_size : 0; in __ksize()
4199 return size; in __ksize()