1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Slab allocator functions that are independent of the allocator strategy
4 *
5 * (C) 2012 Christoph Lameter <cl@linux.com>
6 */
7 #include <linux/slab.h>
8
9 #include <linux/mm.h>
10 #include <linux/poison.h>
11 #include <linux/interrupt.h>
12 #include <linux/memory.h>
13 #include <linux/cache.h>
14 #include <linux/compiler.h>
15 #include <linux/kfence.h>
16 #include <linux/module.h>
17 #include <linux/cpu.h>
18 #include <linux/uaccess.h>
19 #include <linux/seq_file.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/swiotlb.h>
22 #include <linux/proc_fs.h>
23 #include <linux/debugfs.h>
24 #include <linux/kmemleak.h>
25 #include <linux/kasan.h>
26 #include <asm/cacheflush.h>
27 #include <asm/tlbflush.h>
28 #include <asm/page.h>
29 #include <linux/memcontrol.h>
30 #include <linux/stackdepot.h>
31
32 #include "internal.h"
33 #include "slab.h"
34
35 #define CREATE_TRACE_POINTS
36 #include <trace/events/kmem.h>
37 #undef CREATE_TRACE_POINTS
38 #include <trace/hooks/mm.h>
39
40 enum slab_state slab_state;
41 LIST_HEAD(slab_caches);
42 DEFINE_MUTEX(slab_mutex);
43 struct kmem_cache *kmem_cache;
44
45 /*
46 * Set of flags that will prevent slab merging
47 */
48 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
49 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
50 SLAB_FAILSLAB | SLAB_NO_MERGE)
51
52 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
53 SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
54
55 /*
56 * Merge control. If this is set then no merging of slab caches will occur.
57 */
58 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
59
setup_slab_nomerge(char * str)60 static int __init setup_slab_nomerge(char *str)
61 {
62 slab_nomerge = true;
63 return 1;
64 }
65
setup_slab_merge(char * str)66 static int __init setup_slab_merge(char *str)
67 {
68 slab_nomerge = false;
69 return 1;
70 }
71
72 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
73 __setup_param("slub_merge", slub_merge, setup_slab_merge, 0);
74
75 __setup("slab_nomerge", setup_slab_nomerge);
76 __setup("slab_merge", setup_slab_merge);
77
78 /*
79 * Determine the size of a slab object
80 */
kmem_cache_size(struct kmem_cache * s)81 unsigned int kmem_cache_size(struct kmem_cache *s)
82 {
83 return s->object_size;
84 }
85 EXPORT_SYMBOL(kmem_cache_size);
86
87 #ifdef CONFIG_DEBUG_VM
88
kmem_cache_is_duplicate_name(const char * name)89 static bool kmem_cache_is_duplicate_name(const char *name)
90 {
91 struct kmem_cache *s;
92
93 list_for_each_entry(s, &slab_caches, list) {
94 if (!strcmp(s->name, name))
95 return true;
96 }
97
98 return false;
99 }
100
kmem_cache_sanity_check(const char * name,unsigned int size)101 static int kmem_cache_sanity_check(const char *name, unsigned int size)
102 {
103 if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
104 pr_err("kmem_cache_create(%s) integrity check failed\n", name);
105 return -EINVAL;
106 }
107
108 /* Duplicate names will confuse slabtop, et al */
109 WARN(kmem_cache_is_duplicate_name(name),
110 "kmem_cache of name '%s' already exists\n", name);
111
112 WARN_ON(strchr(name, ' ')); /* It confuses parsers */
113 return 0;
114 }
115 #else
kmem_cache_sanity_check(const char * name,unsigned int size)116 static inline int kmem_cache_sanity_check(const char *name, unsigned int size)
117 {
118 return 0;
119 }
120 #endif
121
122 /*
123 * Figure out what the alignment of the objects will be given a set of
124 * flags, a user specified alignment and the size of the objects.
125 */
calculate_alignment(slab_flags_t flags,unsigned int align,unsigned int size)126 static unsigned int calculate_alignment(slab_flags_t flags,
127 unsigned int align, unsigned int size)
128 {
129 /*
130 * If the user wants hardware cache aligned objects then follow that
131 * suggestion if the object is sufficiently large.
132 *
133 * The hardware cache alignment cannot override the specified
134 * alignment though. If that is greater then use it.
135 */
136 if (flags & SLAB_HWCACHE_ALIGN) {
137 unsigned int ralign;
138
139 ralign = cache_line_size();
140 while (size <= ralign / 2)
141 ralign /= 2;
142 align = max(align, ralign);
143 }
144
145 align = max(align, arch_slab_minalign());
146
147 return ALIGN(align, sizeof(void *));
148 }
149
150 /*
151 * Find a mergeable slab cache
152 */
slab_unmergeable(struct kmem_cache * s)153 int slab_unmergeable(struct kmem_cache *s)
154 {
155 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
156 return 1;
157
158 if (s->ctor)
159 return 1;
160
161 #ifdef CONFIG_HARDENED_USERCOPY
162 if (s->usersize)
163 return 1;
164 #endif
165
166 /*
167 * We may have set a slab to be unmergeable during bootstrap.
168 */
169 if (s->refcount < 0)
170 return 1;
171
172 return 0;
173 }
174
find_mergeable(unsigned int size,unsigned int align,slab_flags_t flags,const char * name,void (* ctor)(void *))175 struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
176 slab_flags_t flags, const char *name, void (*ctor)(void *))
177 {
178 struct kmem_cache *s;
179
180 if (slab_nomerge)
181 return NULL;
182
183 if (ctor)
184 return NULL;
185
186 flags = kmem_cache_flags(flags, name);
187
188 if (flags & SLAB_NEVER_MERGE)
189 return NULL;
190
191 size = ALIGN(size, sizeof(void *));
192 align = calculate_alignment(flags, align, size);
193 size = ALIGN(size, align);
194
195 list_for_each_entry_reverse(s, &slab_caches, list) {
196 if (slab_unmergeable(s))
197 continue;
198
199 if (size > s->size)
200 continue;
201
202 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
203 continue;
204 /*
205 * Check if alignment is compatible.
206 * Courtesy of Adrian Drzewiecki
207 */
208 if ((s->size & ~(align - 1)) != s->size)
209 continue;
210
211 if (s->size - size >= sizeof(void *))
212 continue;
213
214 return s;
215 }
216 return NULL;
217 }
218
create_cache(const char * name,unsigned int object_size,struct kmem_cache_args * args,slab_flags_t flags)219 static struct kmem_cache *create_cache(const char *name,
220 unsigned int object_size,
221 struct kmem_cache_args *args,
222 slab_flags_t flags)
223 {
224 struct kmem_cache *s;
225 int err;
226
227 if (WARN_ON(args->useroffset + args->usersize > object_size))
228 args->useroffset = args->usersize = 0;
229
230 /* If a custom freelist pointer is requested make sure it's sane. */
231 err = -EINVAL;
232 if (args->use_freeptr_offset &&
233 (args->freeptr_offset >= object_size ||
234 !(flags & SLAB_TYPESAFE_BY_RCU) ||
235 !IS_ALIGNED(args->freeptr_offset, __alignof__(freeptr_t))))
236 goto out;
237
238 err = -ENOMEM;
239 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
240 if (!s)
241 goto out;
242 err = do_kmem_cache_create(s, name, object_size, args, flags);
243 if (err)
244 goto out_free_cache;
245
246 s->refcount = 1;
247 list_add(&s->list, &slab_caches);
248 return s;
249
250 out_free_cache:
251 kmem_cache_free(kmem_cache, s);
252 out:
253 return ERR_PTR(err);
254 }
255
256 /**
257 * __kmem_cache_create_args - Create a kmem cache.
258 * @name: A string which is used in /proc/slabinfo to identify this cache.
259 * @object_size: The size of objects to be created in this cache.
260 * @args: Additional arguments for the cache creation (see
261 * &struct kmem_cache_args).
262 * @flags: See %SLAB_* flags for an explanation of individual @flags.
263 *
264 * Not to be called directly, use the kmem_cache_create() wrapper with the same
265 * parameters.
266 *
267 * Context: Cannot be called within a interrupt, but can be interrupted.
268 *
269 * Return: a pointer to the cache on success, NULL on failure.
270 */
__kmem_cache_create_args(const char * name,unsigned int object_size,struct kmem_cache_args * args,slab_flags_t flags)271 struct kmem_cache *__kmem_cache_create_args(const char *name,
272 unsigned int object_size,
273 struct kmem_cache_args *args,
274 slab_flags_t flags)
275 {
276 struct kmem_cache *s = NULL;
277 const char *cache_name;
278 int err;
279
280 #ifdef CONFIG_SLUB_DEBUG
281 /*
282 * If no slab_debug was enabled globally, the static key is not yet
283 * enabled by setup_slub_debug(). Enable it if the cache is being
284 * created with any of the debugging flags passed explicitly.
285 * It's also possible that this is the first cache created with
286 * SLAB_STORE_USER and we should init stack_depot for it.
287 */
288 if (flags & SLAB_DEBUG_FLAGS)
289 static_branch_enable(&slub_debug_enabled);
290 if (flags & SLAB_STORE_USER)
291 stack_depot_init();
292 #endif
293
294 mutex_lock(&slab_mutex);
295
296 err = kmem_cache_sanity_check(name, object_size);
297 if (err) {
298 goto out_unlock;
299 }
300
301 /* Refuse requests with allocator specific flags */
302 if (flags & ~SLAB_FLAGS_PERMITTED) {
303 err = -EINVAL;
304 goto out_unlock;
305 }
306
307 /*
308 * Some allocators will constraint the set of valid flags to a subset
309 * of all flags. We expect them to define CACHE_CREATE_MASK in this
310 * case, and we'll just provide them with a sanitized version of the
311 * passed flags.
312 */
313 flags &= CACHE_CREATE_MASK;
314
315 /* Fail closed on bad usersize of useroffset values. */
316 if (!IS_ENABLED(CONFIG_HARDENED_USERCOPY) ||
317 WARN_ON(!args->usersize && args->useroffset) ||
318 WARN_ON(object_size < args->usersize ||
319 object_size - args->usersize < args->useroffset))
320 args->usersize = args->useroffset = 0;
321
322 if (!args->usersize)
323 s = __kmem_cache_alias(name, object_size, args->align, flags,
324 args->ctor);
325 if (s)
326 goto out_unlock;
327
328 cache_name = kstrdup_const(name, GFP_KERNEL);
329 if (!cache_name) {
330 err = -ENOMEM;
331 goto out_unlock;
332 }
333
334 args->align = calculate_alignment(flags, args->align, object_size);
335 s = create_cache(cache_name, object_size, args, flags);
336 if (IS_ERR(s)) {
337 err = PTR_ERR(s);
338 kfree_const(cache_name);
339 }
340
341 out_unlock:
342 mutex_unlock(&slab_mutex);
343
344 if (err) {
345 if (flags & SLAB_PANIC)
346 panic("%s: Failed to create slab '%s'. Error %d\n",
347 __func__, name, err);
348 else {
349 pr_warn("%s(%s) failed with error %d\n",
350 __func__, name, err);
351 dump_stack();
352 }
353 return NULL;
354 }
355 return s;
356 }
357 EXPORT_SYMBOL(__kmem_cache_create_args);
358
359 static struct kmem_cache *kmem_buckets_cache __ro_after_init;
360
361 /**
362 * kmem_buckets_create - Create a set of caches that handle dynamic sized
363 * allocations via kmem_buckets_alloc()
364 * @name: A prefix string which is used in /proc/slabinfo to identify this
365 * cache. The individual caches with have their sizes as the suffix.
366 * @flags: SLAB flags (see kmem_cache_create() for details).
367 * @useroffset: Starting offset within an allocation that may be copied
368 * to/from userspace.
369 * @usersize: How many bytes, starting at @useroffset, may be copied
370 * to/from userspace.
371 * @ctor: A constructor for the objects, run when new allocations are made.
372 *
373 * Cannot be called within an interrupt, but can be interrupted.
374 *
375 * Return: a pointer to the cache on success, NULL on failure. When
376 * CONFIG_SLAB_BUCKETS is not enabled, ZERO_SIZE_PTR is returned, and
377 * subsequent calls to kmem_buckets_alloc() will fall back to kmalloc().
378 * (i.e. callers only need to check for NULL on failure.)
379 */
kmem_buckets_create(const char * name,slab_flags_t flags,unsigned int useroffset,unsigned int usersize,void (* ctor)(void *))380 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
381 unsigned int useroffset,
382 unsigned int usersize,
383 void (*ctor)(void *))
384 {
385 unsigned long mask = 0;
386 unsigned int idx;
387 kmem_buckets *b;
388
389 BUILD_BUG_ON(ARRAY_SIZE(kmalloc_caches[KMALLOC_NORMAL]) > BITS_PER_LONG);
390
391 /*
392 * When the separate buckets API is not built in, just return
393 * a non-NULL value for the kmem_buckets pointer, which will be
394 * unused when performing allocations.
395 */
396 if (!IS_ENABLED(CONFIG_SLAB_BUCKETS))
397 return ZERO_SIZE_PTR;
398
399 if (WARN_ON(!kmem_buckets_cache))
400 return NULL;
401
402 b = kmem_cache_alloc(kmem_buckets_cache, GFP_KERNEL|__GFP_ZERO);
403 if (WARN_ON(!b))
404 return NULL;
405
406 flags |= SLAB_NO_MERGE;
407
408 for (idx = 0; idx < ARRAY_SIZE(kmalloc_caches[KMALLOC_NORMAL]); idx++) {
409 char *short_size, *cache_name;
410 unsigned int cache_useroffset, cache_usersize;
411 unsigned int size, aligned_idx;
412
413 if (!kmalloc_caches[KMALLOC_NORMAL][idx])
414 continue;
415
416 size = kmalloc_caches[KMALLOC_NORMAL][idx]->object_size;
417 if (!size)
418 continue;
419
420 short_size = strchr(kmalloc_caches[KMALLOC_NORMAL][idx]->name, '-');
421 if (WARN_ON(!short_size))
422 goto fail;
423
424 if (useroffset >= size) {
425 cache_useroffset = 0;
426 cache_usersize = 0;
427 } else {
428 cache_useroffset = useroffset;
429 cache_usersize = min(size - cache_useroffset, usersize);
430 }
431
432 aligned_idx = __kmalloc_index(size, false);
433 if (!(*b)[aligned_idx]) {
434 cache_name = kasprintf(GFP_KERNEL, "%s-%s", name, short_size + 1);
435 if (WARN_ON(!cache_name))
436 goto fail;
437 (*b)[aligned_idx] = kmem_cache_create_usercopy(cache_name, size,
438 0, flags, cache_useroffset,
439 cache_usersize, ctor);
440 kfree(cache_name);
441 if (WARN_ON(!(*b)[aligned_idx]))
442 goto fail;
443 set_bit(aligned_idx, &mask);
444 }
445 if (idx != aligned_idx)
446 (*b)[idx] = (*b)[aligned_idx];
447 }
448
449 return b;
450
451 fail:
452 for_each_set_bit(idx, &mask, ARRAY_SIZE(kmalloc_caches[KMALLOC_NORMAL]))
453 kmem_cache_destroy((*b)[idx]);
454 kmem_cache_free(kmem_buckets_cache, b);
455
456 return NULL;
457 }
458 EXPORT_SYMBOL(kmem_buckets_create);
459
460 /*
461 * For a given kmem_cache, kmem_cache_destroy() should only be called
462 * once or there will be a use-after-free problem. The actual deletion
463 * and release of the kobject does not need slab_mutex or cpu_hotplug_lock
464 * protection. So they are now done without holding those locks.
465 */
kmem_cache_release(struct kmem_cache * s)466 static void kmem_cache_release(struct kmem_cache *s)
467 {
468 kfence_shutdown_cache(s);
469 if (__is_defined(SLAB_SUPPORTS_SYSFS) && slab_state >= FULL)
470 sysfs_slab_release(s);
471 else
472 slab_kmem_cache_release(s);
473 }
474
slab_kmem_cache_release(struct kmem_cache * s)475 void slab_kmem_cache_release(struct kmem_cache *s)
476 {
477 __kmem_cache_release(s);
478 kfree_const(s->name);
479 kmem_cache_free(kmem_cache, s);
480 }
481
kmem_cache_destroy(struct kmem_cache * s)482 void kmem_cache_destroy(struct kmem_cache *s)
483 {
484 int err;
485
486 if (unlikely(!s) || !kasan_check_byte(s))
487 return;
488
489 /* in-flight kfree_rcu()'s may include objects from our cache */
490 kvfree_rcu_barrier();
491
492 if (IS_ENABLED(CONFIG_SLUB_RCU_DEBUG) &&
493 (s->flags & SLAB_TYPESAFE_BY_RCU)) {
494 /*
495 * Under CONFIG_SLUB_RCU_DEBUG, when objects in a
496 * SLAB_TYPESAFE_BY_RCU slab are freed, SLUB will internally
497 * defer their freeing with call_rcu().
498 * Wait for such call_rcu() invocations here before actually
499 * destroying the cache.
500 *
501 * It doesn't matter that we haven't looked at the slab refcount
502 * yet - slabs with SLAB_TYPESAFE_BY_RCU can't be merged, so
503 * the refcount should be 1 here.
504 */
505 rcu_barrier();
506 }
507
508 cpus_read_lock();
509 mutex_lock(&slab_mutex);
510
511 s->refcount--;
512 if (s->refcount) {
513 mutex_unlock(&slab_mutex);
514 cpus_read_unlock();
515 return;
516 }
517
518 /* free asan quarantined objects */
519 kasan_cache_shutdown(s);
520
521 err = __kmem_cache_shutdown(s);
522 if (!slab_in_kunit_test())
523 WARN(err, "%s %s: Slab cache still has objects when called from %pS",
524 __func__, s->name, (void *)_RET_IP_);
525
526 list_del(&s->list);
527
528 mutex_unlock(&slab_mutex);
529 cpus_read_unlock();
530
531 if (slab_state >= FULL)
532 sysfs_slab_unlink(s);
533 debugfs_slab_release(s);
534
535 if (err)
536 return;
537
538 if (s->flags & SLAB_TYPESAFE_BY_RCU)
539 rcu_barrier();
540
541 kmem_cache_release(s);
542 }
543 EXPORT_SYMBOL(kmem_cache_destroy);
544
545 /**
546 * kmem_cache_shrink - Shrink a cache.
547 * @cachep: The cache to shrink.
548 *
549 * Releases as many slabs as possible for a cache.
550 * To help debugging, a zero exit status indicates all slabs were released.
551 *
552 * Return: %0 if all slabs were released, non-zero otherwise
553 */
kmem_cache_shrink(struct kmem_cache * cachep)554 int kmem_cache_shrink(struct kmem_cache *cachep)
555 {
556 kasan_cache_shrink(cachep);
557
558 return __kmem_cache_shrink(cachep);
559 }
560 EXPORT_SYMBOL(kmem_cache_shrink);
561
slab_is_available(void)562 bool slab_is_available(void)
563 {
564 return slab_state >= UP;
565 }
566
567 #ifdef CONFIG_PRINTK
kmem_obj_info(struct kmem_obj_info * kpp,void * object,struct slab * slab)568 static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
569 {
570 if (__kfence_obj_info(kpp, object, slab))
571 return;
572 __kmem_obj_info(kpp, object, slab);
573 }
574
575 /**
576 * kmem_dump_obj - Print available slab provenance information
577 * @object: slab object for which to find provenance information.
578 *
579 * This function uses pr_cont(), so that the caller is expected to have
580 * printed out whatever preamble is appropriate. The provenance information
581 * depends on the type of object and on how much debugging is enabled.
582 * For a slab-cache object, the fact that it is a slab object is printed,
583 * and, if available, the slab name, return address, and stack trace from
584 * the allocation and last free path of that object.
585 *
586 * Return: %true if the pointer is to a not-yet-freed object from
587 * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer
588 * is to an already-freed object, and %false otherwise.
589 */
kmem_dump_obj(void * object)590 bool kmem_dump_obj(void *object)
591 {
592 char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc";
593 int i;
594 struct slab *slab;
595 unsigned long ptroffset;
596 struct kmem_obj_info kp = { };
597
598 /* Some arches consider ZERO_SIZE_PTR to be a valid address. */
599 if (object < (void *)PAGE_SIZE || !virt_addr_valid(object))
600 return false;
601 slab = virt_to_slab(object);
602 if (!slab)
603 return false;
604
605 kmem_obj_info(&kp, object, slab);
606 if (kp.kp_slab_cache)
607 pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name);
608 else
609 pr_cont(" slab%s", cp);
610 if (is_kfence_address(object))
611 pr_cont(" (kfence)");
612 if (kp.kp_objp)
613 pr_cont(" start %px", kp.kp_objp);
614 if (kp.kp_data_offset)
615 pr_cont(" data offset %lu", kp.kp_data_offset);
616 if (kp.kp_objp) {
617 ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset;
618 pr_cont(" pointer offset %lu", ptroffset);
619 }
620 if (kp.kp_slab_cache && kp.kp_slab_cache->object_size)
621 pr_cont(" size %u", kp.kp_slab_cache->object_size);
622 if (kp.kp_ret)
623 pr_cont(" allocated at %pS\n", kp.kp_ret);
624 else
625 pr_cont("\n");
626 for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) {
627 if (!kp.kp_stack[i])
628 break;
629 pr_info(" %pS\n", kp.kp_stack[i]);
630 }
631
632 if (kp.kp_free_stack[0])
633 pr_cont(" Free path:\n");
634
635 for (i = 0; i < ARRAY_SIZE(kp.kp_free_stack); i++) {
636 if (!kp.kp_free_stack[i])
637 break;
638 pr_info(" %pS\n", kp.kp_free_stack[i]);
639 }
640
641 return true;
642 }
643 EXPORT_SYMBOL_GPL(kmem_dump_obj);
644 #endif
645
646 /* Create a cache during boot when no slab services are available yet */
create_boot_cache(struct kmem_cache * s,const char * name,unsigned int size,slab_flags_t flags,unsigned int useroffset,unsigned int usersize)647 void __init create_boot_cache(struct kmem_cache *s, const char *name,
648 unsigned int size, slab_flags_t flags,
649 unsigned int useroffset, unsigned int usersize)
650 {
651 int err;
652 unsigned int align = ARCH_KMALLOC_MINALIGN;
653 struct kmem_cache_args kmem_args = {};
654
655 /*
656 * kmalloc caches guarantee alignment of at least the largest
657 * power-of-two divisor of the size. For power-of-two sizes,
658 * it is the size itself.
659 */
660 if (flags & SLAB_KMALLOC)
661 align = max(align, 1U << (ffs(size) - 1));
662 kmem_args.align = calculate_alignment(flags, align, size);
663
664 #ifdef CONFIG_HARDENED_USERCOPY
665 kmem_args.useroffset = useroffset;
666 kmem_args.usersize = usersize;
667 #endif
668
669 err = do_kmem_cache_create(s, name, size, &kmem_args, flags);
670
671 if (err)
672 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n",
673 name, size, err);
674
675 s->refcount = -1; /* Exempt from merging for now */
676 }
677
create_kmalloc_cache(const char * name,unsigned int size,slab_flags_t flags)678 static struct kmem_cache *__init create_kmalloc_cache(const char *name,
679 unsigned int size,
680 slab_flags_t flags)
681 {
682 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
683
684 if (!s)
685 panic("Out of memory when creating slab %s\n", name);
686
687 create_boot_cache(s, name, size, flags | SLAB_KMALLOC, 0, size);
688 list_add(&s->list, &slab_caches);
689 s->refcount = 1;
690 return s;
691 }
692
693 kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES] __ro_after_init =
694 { /* initialization for https://llvm.org/pr42570 */ };
695 EXPORT_SYMBOL(kmalloc_caches);
696
697 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
698 unsigned long random_kmalloc_seed __ro_after_init;
699 EXPORT_SYMBOL(random_kmalloc_seed);
700 #endif
701
702 /*
703 * Conversion table for small slabs sizes / 8 to the index in the
704 * kmalloc array. This is necessary for slabs < 192 since we have non power
705 * of two cache sizes there. The size of larger slabs can be determined using
706 * fls.
707 */
708 u8 kmalloc_size_index[24] __ro_after_init = {
709 3, /* 8 */
710 4, /* 16 */
711 5, /* 24 */
712 5, /* 32 */
713 6, /* 40 */
714 6, /* 48 */
715 6, /* 56 */
716 6, /* 64 */
717 1, /* 72 */
718 1, /* 80 */
719 1, /* 88 */
720 1, /* 96 */
721 7, /* 104 */
722 7, /* 112 */
723 7, /* 120 */
724 7, /* 128 */
725 2, /* 136 */
726 2, /* 144 */
727 2, /* 152 */
728 2, /* 160 */
729 2, /* 168 */
730 2, /* 176 */
731 2, /* 184 */
732 2 /* 192 */
733 };
734
kmalloc_size_roundup(size_t size)735 size_t kmalloc_size_roundup(size_t size)
736 {
737 if (size && size <= KMALLOC_MAX_CACHE_SIZE) {
738 /*
739 * The flags don't matter since size_index is common to all.
740 * Neither does the caller for just getting ->object_size.
741 */
742 return kmalloc_slab(size, NULL, GFP_KERNEL, 0)->object_size;
743 }
744
745 /* Above the smaller buckets, size is a multiple of page size. */
746 if (size && size <= KMALLOC_MAX_SIZE)
747 return PAGE_SIZE << get_order(size);
748
749 /*
750 * Return 'size' for 0 - kmalloc() returns ZERO_SIZE_PTR
751 * and very large size - kmalloc() may fail.
752 */
753 return size;
754
755 }
756 EXPORT_SYMBOL(kmalloc_size_roundup);
757
758 #ifdef CONFIG_ZONE_DMA
759 #define KMALLOC_DMA_NAME(sz) .name[KMALLOC_DMA] = "dma-kmalloc-" #sz,
760 #else
761 #define KMALLOC_DMA_NAME(sz)
762 #endif
763
764 #ifdef CONFIG_MEMCG
765 #define KMALLOC_CGROUP_NAME(sz) .name[KMALLOC_CGROUP] = "kmalloc-cg-" #sz,
766 #else
767 #define KMALLOC_CGROUP_NAME(sz)
768 #endif
769
770 #ifndef CONFIG_SLUB_TINY
771 #define KMALLOC_RCL_NAME(sz) .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #sz,
772 #else
773 #define KMALLOC_RCL_NAME(sz)
774 #endif
775
776 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
777 #define __KMALLOC_RANDOM_CONCAT(a, b) a ## b
778 #define KMALLOC_RANDOM_NAME(N, sz) __KMALLOC_RANDOM_CONCAT(KMA_RAND_, N)(sz)
779 #define KMA_RAND_1(sz) .name[KMALLOC_RANDOM_START + 1] = "kmalloc-rnd-01-" #sz,
780 #define KMA_RAND_2(sz) KMA_RAND_1(sz) .name[KMALLOC_RANDOM_START + 2] = "kmalloc-rnd-02-" #sz,
781 #define KMA_RAND_3(sz) KMA_RAND_2(sz) .name[KMALLOC_RANDOM_START + 3] = "kmalloc-rnd-03-" #sz,
782 #define KMA_RAND_4(sz) KMA_RAND_3(sz) .name[KMALLOC_RANDOM_START + 4] = "kmalloc-rnd-04-" #sz,
783 #define KMA_RAND_5(sz) KMA_RAND_4(sz) .name[KMALLOC_RANDOM_START + 5] = "kmalloc-rnd-05-" #sz,
784 #define KMA_RAND_6(sz) KMA_RAND_5(sz) .name[KMALLOC_RANDOM_START + 6] = "kmalloc-rnd-06-" #sz,
785 #define KMA_RAND_7(sz) KMA_RAND_6(sz) .name[KMALLOC_RANDOM_START + 7] = "kmalloc-rnd-07-" #sz,
786 #define KMA_RAND_8(sz) KMA_RAND_7(sz) .name[KMALLOC_RANDOM_START + 8] = "kmalloc-rnd-08-" #sz,
787 #define KMA_RAND_9(sz) KMA_RAND_8(sz) .name[KMALLOC_RANDOM_START + 9] = "kmalloc-rnd-09-" #sz,
788 #define KMA_RAND_10(sz) KMA_RAND_9(sz) .name[KMALLOC_RANDOM_START + 10] = "kmalloc-rnd-10-" #sz,
789 #define KMA_RAND_11(sz) KMA_RAND_10(sz) .name[KMALLOC_RANDOM_START + 11] = "kmalloc-rnd-11-" #sz,
790 #define KMA_RAND_12(sz) KMA_RAND_11(sz) .name[KMALLOC_RANDOM_START + 12] = "kmalloc-rnd-12-" #sz,
791 #define KMA_RAND_13(sz) KMA_RAND_12(sz) .name[KMALLOC_RANDOM_START + 13] = "kmalloc-rnd-13-" #sz,
792 #define KMA_RAND_14(sz) KMA_RAND_13(sz) .name[KMALLOC_RANDOM_START + 14] = "kmalloc-rnd-14-" #sz,
793 #define KMA_RAND_15(sz) KMA_RAND_14(sz) .name[KMALLOC_RANDOM_START + 15] = "kmalloc-rnd-15-" #sz,
794 #else // CONFIG_RANDOM_KMALLOC_CACHES
795 #define KMALLOC_RANDOM_NAME(N, sz)
796 #endif
797
798 #define INIT_KMALLOC_INFO(__size, __short_size) \
799 { \
800 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
801 KMALLOC_RCL_NAME(__short_size) \
802 KMALLOC_CGROUP_NAME(__short_size) \
803 KMALLOC_DMA_NAME(__short_size) \
804 KMALLOC_RANDOM_NAME(RANDOM_KMALLOC_CACHES_NR, __short_size) \
805 .size = __size, \
806 }
807
808 /*
809 * kmalloc_info[] is to make slab_debug=,kmalloc-xx option work at boot time.
810 * kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is
811 * kmalloc-2M.
812 */
813 const struct kmalloc_info_struct kmalloc_info[] __initconst = {
814 INIT_KMALLOC_INFO(0, 0),
815 INIT_KMALLOC_INFO(96, 96),
816 INIT_KMALLOC_INFO(192, 192),
817 INIT_KMALLOC_INFO(8, 8),
818 INIT_KMALLOC_INFO(16, 16),
819 INIT_KMALLOC_INFO(32, 32),
820 INIT_KMALLOC_INFO(64, 64),
821 INIT_KMALLOC_INFO(128, 128),
822 INIT_KMALLOC_INFO(256, 256),
823 INIT_KMALLOC_INFO(512, 512),
824 INIT_KMALLOC_INFO(1024, 1k),
825 INIT_KMALLOC_INFO(2048, 2k),
826 INIT_KMALLOC_INFO(4096, 4k),
827 INIT_KMALLOC_INFO(8192, 8k),
828 INIT_KMALLOC_INFO(16384, 16k),
829 INIT_KMALLOC_INFO(32768, 32k),
830 INIT_KMALLOC_INFO(65536, 64k),
831 INIT_KMALLOC_INFO(131072, 128k),
832 INIT_KMALLOC_INFO(262144, 256k),
833 INIT_KMALLOC_INFO(524288, 512k),
834 INIT_KMALLOC_INFO(1048576, 1M),
835 INIT_KMALLOC_INFO(2097152, 2M)
836 };
837
838 /*
839 * Patch up the size_index table if we have strange large alignment
840 * requirements for the kmalloc array. This is only the case for
841 * MIPS it seems. The standard arches will not generate any code here.
842 *
843 * Largest permitted alignment is 256 bytes due to the way we
844 * handle the index determination for the smaller caches.
845 *
846 * Make sure that nothing crazy happens if someone starts tinkering
847 * around with ARCH_KMALLOC_MINALIGN
848 */
setup_kmalloc_cache_index_table(void)849 void __init setup_kmalloc_cache_index_table(void)
850 {
851 unsigned int i;
852
853 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
854 !is_power_of_2(KMALLOC_MIN_SIZE));
855
856 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
857 unsigned int elem = size_index_elem(i);
858
859 if (elem >= ARRAY_SIZE(kmalloc_size_index))
860 break;
861 kmalloc_size_index[elem] = KMALLOC_SHIFT_LOW;
862 }
863
864 if (KMALLOC_MIN_SIZE >= 64) {
865 /*
866 * The 96 byte sized cache is not used if the alignment
867 * is 64 byte.
868 */
869 for (i = 64 + 8; i <= 96; i += 8)
870 kmalloc_size_index[size_index_elem(i)] = 7;
871
872 }
873
874 if (KMALLOC_MIN_SIZE >= 128) {
875 /*
876 * The 192 byte sized cache is not used if the alignment
877 * is 128 byte. Redirect kmalloc to use the 256 byte cache
878 * instead.
879 */
880 for (i = 128 + 8; i <= 192; i += 8)
881 kmalloc_size_index[size_index_elem(i)] = 8;
882 }
883 }
884
__kmalloc_minalign(void)885 static unsigned int __kmalloc_minalign(void)
886 {
887 unsigned int minalign = dma_get_cache_alignment();
888
889 if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) &&
890 is_swiotlb_allocated())
891 minalign = ARCH_KMALLOC_MINALIGN;
892
893 return max(minalign, arch_slab_minalign());
894 }
895
896 static void __init
new_kmalloc_cache(int idx,enum kmalloc_cache_type type)897 new_kmalloc_cache(int idx, enum kmalloc_cache_type type)
898 {
899 slab_flags_t flags = 0;
900 unsigned int minalign = __kmalloc_minalign();
901 unsigned int aligned_size = kmalloc_info[idx].size;
902 int aligned_idx = idx;
903
904 if ((KMALLOC_RECLAIM != KMALLOC_NORMAL) && (type == KMALLOC_RECLAIM)) {
905 flags |= SLAB_RECLAIM_ACCOUNT;
906 } else if (IS_ENABLED(CONFIG_MEMCG) && (type == KMALLOC_CGROUP)) {
907 if (mem_cgroup_kmem_disabled()) {
908 kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
909 return;
910 }
911 flags |= SLAB_ACCOUNT;
912 } else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) {
913 flags |= SLAB_CACHE_DMA;
914 }
915
916 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
917 if (type >= KMALLOC_RANDOM_START && type <= KMALLOC_RANDOM_END)
918 flags |= SLAB_NO_MERGE;
919 #endif
920
921 /*
922 * If CONFIG_MEMCG is enabled, disable cache merging for
923 * KMALLOC_NORMAL caches.
924 */
925 if (IS_ENABLED(CONFIG_MEMCG) && (type == KMALLOC_NORMAL))
926 flags |= SLAB_NO_MERGE;
927
928 if (minalign > ARCH_KMALLOC_MINALIGN) {
929 aligned_size = ALIGN(aligned_size, minalign);
930 aligned_idx = __kmalloc_index(aligned_size, false);
931 }
932
933 if (!kmalloc_caches[type][aligned_idx])
934 kmalloc_caches[type][aligned_idx] = create_kmalloc_cache(
935 kmalloc_info[aligned_idx].name[type],
936 aligned_size, flags);
937 if (idx != aligned_idx)
938 kmalloc_caches[type][idx] = kmalloc_caches[type][aligned_idx];
939 }
940
941 /*
942 * Create the kmalloc array. Some of the regular kmalloc arrays
943 * may already have been created because they were needed to
944 * enable allocations for slab creation.
945 */
create_kmalloc_caches(void)946 void __init create_kmalloc_caches(void)
947 {
948 int i;
949 enum kmalloc_cache_type type;
950
951 /*
952 * Including KMALLOC_CGROUP if CONFIG_MEMCG defined
953 */
954 for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
955 /* Caches that are NOT of the two-to-the-power-of size. */
956 if (KMALLOC_MIN_SIZE <= 32)
957 new_kmalloc_cache(1, type);
958 if (KMALLOC_MIN_SIZE <= 64)
959 new_kmalloc_cache(2, type);
960
961 /* Caches that are of the two-to-the-power-of size. */
962 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
963 new_kmalloc_cache(i, type);
964 }
965 #ifdef CONFIG_RANDOM_KMALLOC_CACHES
966 random_kmalloc_seed = get_random_u64();
967 #endif
968
969 /* Kmalloc array is now usable */
970 slab_state = UP;
971
972 if (IS_ENABLED(CONFIG_SLAB_BUCKETS))
973 kmem_buckets_cache = kmem_cache_create("kmalloc_buckets",
974 sizeof(kmem_buckets),
975 0, SLAB_NO_MERGE, NULL);
976 }
977
978 /**
979 * __ksize -- Report full size of underlying allocation
980 * @object: pointer to the object
981 *
982 * This should only be used internally to query the true size of allocations.
983 * It is not meant to be a way to discover the usable size of an allocation
984 * after the fact. Instead, use kmalloc_size_roundup(). Using memory beyond
985 * the originally requested allocation size may trigger KASAN, UBSAN_BOUNDS,
986 * and/or FORTIFY_SOURCE.
987 *
988 * Return: size of the actual memory used by @object in bytes
989 */
__ksize(const void * object)990 size_t __ksize(const void *object)
991 {
992 struct folio *folio;
993 size_t up_size = 0;
994
995 if (unlikely(object == ZERO_SIZE_PTR))
996 return 0;
997
998 folio = virt_to_folio(object);
999
1000 if (unlikely(!folio_test_slab(folio))) {
1001 trace_android_vh_ksize(folio, &up_size);
1002 if (up_size)
1003 return up_size;
1004
1005 if (WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE))
1006 return 0;
1007 if (WARN_ON(object != folio_address(folio)))
1008 return 0;
1009
1010 return folio_size(folio);
1011 }
1012
1013 #ifdef CONFIG_SLUB_DEBUG
1014 skip_orig_size_check(folio_slab(folio)->slab_cache, object);
1015 #endif
1016
1017 return slab_ksize(folio_slab(folio)->slab_cache);
1018 }
1019
kmalloc_fix_flags(gfp_t flags)1020 gfp_t kmalloc_fix_flags(gfp_t flags)
1021 {
1022 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
1023
1024 flags &= ~GFP_SLAB_BUG_MASK;
1025 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
1026 invalid_mask, &invalid_mask, flags, &flags);
1027 dump_stack();
1028
1029 return flags;
1030 }
1031
1032 #ifdef CONFIG_SLAB_FREELIST_RANDOM
1033 /* Randomize a generic freelist */
freelist_randomize(unsigned int * list,unsigned int count)1034 static void freelist_randomize(unsigned int *list,
1035 unsigned int count)
1036 {
1037 unsigned int rand;
1038 unsigned int i;
1039
1040 for (i = 0; i < count; i++)
1041 list[i] = i;
1042
1043 /* Fisher-Yates shuffle */
1044 for (i = count - 1; i > 0; i--) {
1045 rand = get_random_u32_below(i + 1);
1046 swap(list[i], list[rand]);
1047 }
1048 }
1049
1050 /* Create a random sequence per cache */
cache_random_seq_create(struct kmem_cache * cachep,unsigned int count,gfp_t gfp)1051 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
1052 gfp_t gfp)
1053 {
1054
1055 if (count < 2 || cachep->random_seq)
1056 return 0;
1057
1058 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp);
1059 if (!cachep->random_seq)
1060 return -ENOMEM;
1061
1062 freelist_randomize(cachep->random_seq, count);
1063 return 0;
1064 }
1065
1066 /* Destroy the per-cache random freelist sequence */
cache_random_seq_destroy(struct kmem_cache * cachep)1067 void cache_random_seq_destroy(struct kmem_cache *cachep)
1068 {
1069 kfree(cachep->random_seq);
1070 cachep->random_seq = NULL;
1071 }
1072 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
1073
1074 #ifdef CONFIG_SLUB_DEBUG
1075 #define SLABINFO_RIGHTS (0400)
1076
print_slabinfo_header(struct seq_file * m)1077 static void print_slabinfo_header(struct seq_file *m)
1078 {
1079 /*
1080 * Output format version, so at least we can change it
1081 * without _too_ many complaints.
1082 */
1083 seq_puts(m, "slabinfo - version: 2.1\n");
1084 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
1085 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
1086 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
1087 trace_android_vh_print_slabinfo_header(m);
1088 seq_putc(m, '\n');
1089 }
1090
slab_start(struct seq_file * m,loff_t * pos)1091 static void *slab_start(struct seq_file *m, loff_t *pos)
1092 {
1093 mutex_lock(&slab_mutex);
1094 return seq_list_start(&slab_caches, *pos);
1095 }
1096
slab_next(struct seq_file * m,void * p,loff_t * pos)1097 static void *slab_next(struct seq_file *m, void *p, loff_t *pos)
1098 {
1099 return seq_list_next(p, &slab_caches, pos);
1100 }
1101
slab_stop(struct seq_file * m,void * p)1102 static void slab_stop(struct seq_file *m, void *p)
1103 {
1104 mutex_unlock(&slab_mutex);
1105 }
1106
cache_show(struct kmem_cache * s,struct seq_file * m)1107 static void cache_show(struct kmem_cache *s, struct seq_file *m)
1108 {
1109 struct slabinfo sinfo;
1110
1111 memset(&sinfo, 0, sizeof(sinfo));
1112 get_slabinfo(s, &sinfo);
1113
1114 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1115 s->name, sinfo.active_objs, sinfo.num_objs, s->size,
1116 sinfo.objects_per_slab, (1 << sinfo.cache_order));
1117
1118 seq_printf(m, " : tunables %4u %4u %4u",
1119 sinfo.limit, sinfo.batchcount, sinfo.shared);
1120 seq_printf(m, " : slabdata %6lu %6lu %6lu",
1121 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
1122 trace_android_vh_cache_show(m, &sinfo, s);
1123 seq_putc(m, '\n');
1124 }
1125
slab_show(struct seq_file * m,void * p)1126 static int slab_show(struct seq_file *m, void *p)
1127 {
1128 struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
1129
1130 if (p == slab_caches.next)
1131 print_slabinfo_header(m);
1132 cache_show(s, m);
1133 return 0;
1134 }
1135
dump_unreclaimable_slab(void)1136 void dump_unreclaimable_slab(void)
1137 {
1138 struct kmem_cache *s;
1139 struct slabinfo sinfo;
1140
1141 /*
1142 * Here acquiring slab_mutex is risky since we don't prefer to get
1143 * sleep in oom path. But, without mutex hold, it may introduce a
1144 * risk of crash.
1145 * Use mutex_trylock to protect the list traverse, dump nothing
1146 * without acquiring the mutex.
1147 */
1148 if (!mutex_trylock(&slab_mutex)) {
1149 pr_warn("excessive unreclaimable slab but cannot dump stats\n");
1150 return;
1151 }
1152
1153 pr_info("Unreclaimable slab info:\n");
1154 pr_info("Name Used Total\n");
1155
1156 list_for_each_entry(s, &slab_caches, list) {
1157 if (s->flags & SLAB_RECLAIM_ACCOUNT)
1158 continue;
1159
1160 get_slabinfo(s, &sinfo);
1161
1162 if (sinfo.num_objs > 0)
1163 pr_info("%-17s %10luKB %10luKB\n", s->name,
1164 (sinfo.active_objs * s->size) / 1024,
1165 (sinfo.num_objs * s->size) / 1024);
1166 }
1167 mutex_unlock(&slab_mutex);
1168 }
1169
1170 /*
1171 * slabinfo_op - iterator that generates /proc/slabinfo
1172 *
1173 * Output layout:
1174 * cache-name
1175 * num-active-objs
1176 * total-objs
1177 * object size
1178 * num-active-slabs
1179 * total-slabs
1180 * num-pages-per-slab
1181 * + further values on SMP and with statistics enabled
1182 */
1183 static const struct seq_operations slabinfo_op = {
1184 .start = slab_start,
1185 .next = slab_next,
1186 .stop = slab_stop,
1187 .show = slab_show,
1188 };
1189
slabinfo_open(struct inode * inode,struct file * file)1190 static int slabinfo_open(struct inode *inode, struct file *file)
1191 {
1192 return seq_open(file, &slabinfo_op);
1193 }
1194
1195 static const struct proc_ops slabinfo_proc_ops = {
1196 .proc_flags = PROC_ENTRY_PERMANENT,
1197 .proc_open = slabinfo_open,
1198 .proc_read = seq_read,
1199 .proc_lseek = seq_lseek,
1200 .proc_release = seq_release,
1201 };
1202
slab_proc_init(void)1203 static int __init slab_proc_init(void)
1204 {
1205 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, &slabinfo_proc_ops);
1206 return 0;
1207 }
1208 module_init(slab_proc_init);
1209
1210 #endif /* CONFIG_SLUB_DEBUG */
1211
1212 static __always_inline __realloc_size(2) void *
__do_krealloc(const void * p,size_t new_size,gfp_t flags)1213 __do_krealloc(const void *p, size_t new_size, gfp_t flags)
1214 {
1215 void *ret;
1216 size_t ks;
1217
1218 /* Check for double-free before calling ksize. */
1219 if (likely(!ZERO_OR_NULL_PTR(p))) {
1220 if (!kasan_check_byte(p))
1221 return NULL;
1222 ks = ksize(p);
1223 } else
1224 ks = 0;
1225
1226 /* If the object still fits, repoison it precisely. */
1227 if (ks >= new_size) {
1228 /* Zero out spare memory. */
1229 if (want_init_on_alloc(flags)) {
1230 kasan_disable_current();
1231 memset(kasan_reset_tag(p) + new_size, 0, ks - new_size);
1232 kasan_enable_current();
1233 }
1234
1235 p = kasan_krealloc((void *)p, new_size, flags);
1236 return (void *)p;
1237 }
1238
1239 ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_);
1240 if (ret && p) {
1241 /* Disable KASAN checks as the object's redzone is accessed. */
1242 kasan_disable_current();
1243 memcpy(ret, kasan_reset_tag(p), ks);
1244 kasan_enable_current();
1245 }
1246
1247 return ret;
1248 }
1249
1250 /**
1251 * krealloc - reallocate memory. The contents will remain unchanged.
1252 * @p: object to reallocate memory for.
1253 * @new_size: how many bytes of memory are required.
1254 * @flags: the type of memory to allocate.
1255 *
1256 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size
1257 * is 0 and @p is not a %NULL pointer, the object pointed to is freed.
1258 *
1259 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
1260 * initial memory allocation, every subsequent call to this API for the same
1261 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
1262 * __GFP_ZERO is not fully honored by this API.
1263 *
1264 * This is the case, since krealloc() only knows about the bucket size of an
1265 * allocation (but not the exact size it was allocated with) and hence
1266 * implements the following semantics for shrinking and growing buffers with
1267 * __GFP_ZERO.
1268 *
1269 * new bucket
1270 * 0 size size
1271 * |--------|----------------|
1272 * | keep | zero |
1273 *
1274 * In any case, the contents of the object pointed to are preserved up to the
1275 * lesser of the new and old sizes.
1276 *
1277 * Return: pointer to the allocated memory or %NULL in case of error
1278 */
krealloc_noprof(const void * p,size_t new_size,gfp_t flags)1279 void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)
1280 {
1281 void *ret;
1282
1283 if (unlikely(!new_size)) {
1284 kfree(p);
1285 return ZERO_SIZE_PTR;
1286 }
1287
1288 ret = __do_krealloc(p, new_size, flags);
1289 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
1290 kfree(p);
1291
1292 return ret;
1293 }
1294 EXPORT_SYMBOL(krealloc_noprof);
1295
1296 /**
1297 * kfree_sensitive - Clear sensitive information in memory before freeing
1298 * @p: object to free memory of
1299 *
1300 * The memory of the object @p points to is zeroed before freed.
1301 * If @p is %NULL, kfree_sensitive() does nothing.
1302 *
1303 * Note: this function zeroes the whole allocated buffer which can be a good
1304 * deal bigger than the requested buffer size passed to kmalloc(). So be
1305 * careful when using this function in performance sensitive code.
1306 */
kfree_sensitive(const void * p)1307 void kfree_sensitive(const void *p)
1308 {
1309 size_t ks;
1310 void *mem = (void *)p;
1311
1312 ks = ksize(mem);
1313 if (ks) {
1314 kasan_unpoison_range(mem, ks);
1315 memzero_explicit(mem, ks);
1316 }
1317 kfree(mem);
1318 }
1319 EXPORT_SYMBOL(kfree_sensitive);
1320
ksize(const void * objp)1321 size_t ksize(const void *objp)
1322 {
1323 /*
1324 * We need to first check that the pointer to the object is valid.
1325 * The KASAN report printed from ksize() is more useful, then when
1326 * it's printed later when the behaviour could be undefined due to
1327 * a potential use-after-free or double-free.
1328 *
1329 * We use kasan_check_byte(), which is supported for the hardware
1330 * tag-based KASAN mode, unlike kasan_check_read/write().
1331 *
1332 * If the pointed to memory is invalid, we return 0 to avoid users of
1333 * ksize() writing to and potentially corrupting the memory region.
1334 *
1335 * We want to perform the check before __ksize(), to avoid potentially
1336 * crashing in __ksize() due to accessing invalid metadata.
1337 */
1338 if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp))
1339 return 0;
1340
1341 return kfence_ksize(objp) ?: __ksize(objp);
1342 }
1343 EXPORT_SYMBOL(ksize);
1344
1345 /* Tracepoints definitions. */
1346 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1347 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
1348 EXPORT_TRACEPOINT_SYMBOL(kfree);
1349 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
1350
1351