1 #define JEMALLOC_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /******************************************************************************/
5 /* Data. */
6
7 /* Runtime configuration options. */
8 const char *je_malloc_conf
9 #ifndef _WIN32
10 JEMALLOC_ATTR(weak)
11 #endif
12 ;
13 bool opt_abort =
14 #ifdef JEMALLOC_DEBUG
15 true
16 #else
17 false
18 #endif
19 ;
20 const char *opt_junk =
21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
22 "true"
23 #else
24 "false"
25 #endif
26 ;
27 bool opt_junk_alloc =
28 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
29 true
30 #else
31 false
32 #endif
33 ;
34 bool opt_junk_free =
35 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
36 true
37 #else
38 false
39 #endif
40 ;
41
42 size_t opt_quarantine = ZU(0);
43 bool opt_redzone = false;
44 bool opt_utrace = false;
45 bool opt_xmalloc = false;
46 bool opt_zero = false;
47 unsigned opt_narenas = 0;
48
49 /* Initialized to true if the process is running inside Valgrind. */
50 bool in_valgrind;
51
52 unsigned ncpus;
53
54 /* Protects arenas initialization. */
55 static malloc_mutex_t arenas_lock;
56 /*
57 * Arenas that are used to service external requests. Not all elements of the
58 * arenas array are necessarily used; arenas are created lazily as needed.
59 *
60 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
61 * arenas. arenas[narenas_auto..narenas_total) are only used if the application
62 * takes some action to create them and allocate from them.
63 */
64 arena_t **arenas;
65 static unsigned narenas_total; /* Use narenas_total_*(). */
66 static arena_t *a0; /* arenas[0]; read-only after initialization. */
67 unsigned narenas_auto; /* Read-only after initialization. */
68
69 typedef enum {
70 malloc_init_uninitialized = 3,
71 malloc_init_a0_initialized = 2,
72 malloc_init_recursible = 1,
73 malloc_init_initialized = 0 /* Common case --> jnz. */
74 } malloc_init_t;
75 static malloc_init_t malloc_init_state = malloc_init_uninitialized;
76
77 /* False should be the common case. Set to true to trigger initialization. */
78 static bool malloc_slow = true;
79
80 /* When malloc_slow is true, set the corresponding bits for sanity check. */
81 enum {
82 flag_opt_junk_alloc = (1U),
83 flag_opt_junk_free = (1U << 1),
84 flag_opt_quarantine = (1U << 2),
85 flag_opt_zero = (1U << 3),
86 flag_opt_utrace = (1U << 4),
87 flag_in_valgrind = (1U << 5),
88 flag_opt_xmalloc = (1U << 6)
89 };
90 static uint8_t malloc_slow_flags;
91
92 JEMALLOC_ALIGNED(CACHELINE)
93 const size_t pind2sz_tab[NPSIZES] = {
94 #define PSZ_yes(lg_grp, ndelta, lg_delta) \
95 (((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))),
96 #define PSZ_no(lg_grp, ndelta, lg_delta)
97 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
98 PSZ_##psz(lg_grp, ndelta, lg_delta)
99 SIZE_CLASSES
100 #undef PSZ_yes
101 #undef PSZ_no
102 #undef SC
103 };
104
105 JEMALLOC_ALIGNED(CACHELINE)
106 const size_t index2size_tab[NSIZES] = {
107 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
108 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
109 SIZE_CLASSES
110 #undef SC
111 };
112
113 JEMALLOC_ALIGNED(CACHELINE)
114 const uint8_t size2index_tab[] = {
115 #if LG_TINY_MIN == 0
116 #warning "Dangerous LG_TINY_MIN"
117 #define S2B_0(i) i,
118 #elif LG_TINY_MIN == 1
119 #warning "Dangerous LG_TINY_MIN"
120 #define S2B_1(i) i,
121 #elif LG_TINY_MIN == 2
122 #warning "Dangerous LG_TINY_MIN"
123 #define S2B_2(i) i,
124 #elif LG_TINY_MIN == 3
125 #define S2B_3(i) i,
126 #elif LG_TINY_MIN == 4
127 #define S2B_4(i) i,
128 #elif LG_TINY_MIN == 5
129 #define S2B_5(i) i,
130 #elif LG_TINY_MIN == 6
131 #define S2B_6(i) i,
132 #elif LG_TINY_MIN == 7
133 #define S2B_7(i) i,
134 #elif LG_TINY_MIN == 8
135 #define S2B_8(i) i,
136 #elif LG_TINY_MIN == 9
137 #define S2B_9(i) i,
138 #elif LG_TINY_MIN == 10
139 #define S2B_10(i) i,
140 #elif LG_TINY_MIN == 11
141 #define S2B_11(i) i,
142 #else
143 #error "Unsupported LG_TINY_MIN"
144 #endif
145 #if LG_TINY_MIN < 1
146 #define S2B_1(i) S2B_0(i) S2B_0(i)
147 #endif
148 #if LG_TINY_MIN < 2
149 #define S2B_2(i) S2B_1(i) S2B_1(i)
150 #endif
151 #if LG_TINY_MIN < 3
152 #define S2B_3(i) S2B_2(i) S2B_2(i)
153 #endif
154 #if LG_TINY_MIN < 4
155 #define S2B_4(i) S2B_3(i) S2B_3(i)
156 #endif
157 #if LG_TINY_MIN < 5
158 #define S2B_5(i) S2B_4(i) S2B_4(i)
159 #endif
160 #if LG_TINY_MIN < 6
161 #define S2B_6(i) S2B_5(i) S2B_5(i)
162 #endif
163 #if LG_TINY_MIN < 7
164 #define S2B_7(i) S2B_6(i) S2B_6(i)
165 #endif
166 #if LG_TINY_MIN < 8
167 #define S2B_8(i) S2B_7(i) S2B_7(i)
168 #endif
169 #if LG_TINY_MIN < 9
170 #define S2B_9(i) S2B_8(i) S2B_8(i)
171 #endif
172 #if LG_TINY_MIN < 10
173 #define S2B_10(i) S2B_9(i) S2B_9(i)
174 #endif
175 #if LG_TINY_MIN < 11
176 #define S2B_11(i) S2B_10(i) S2B_10(i)
177 #endif
178 #define S2B_no(i)
179 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
180 S2B_##lg_delta_lookup(index)
181 SIZE_CLASSES
182 #undef S2B_3
183 #undef S2B_4
184 #undef S2B_5
185 #undef S2B_6
186 #undef S2B_7
187 #undef S2B_8
188 #undef S2B_9
189 #undef S2B_10
190 #undef S2B_11
191 #undef S2B_no
192 #undef SC
193 };
194
195 #ifdef JEMALLOC_THREADED_INIT
196 /* Used to let the initializing thread recursively allocate. */
197 # define NO_INITIALIZER ((unsigned long)0)
198 # define INITIALIZER pthread_self()
199 # define IS_INITIALIZER (malloc_initializer == pthread_self())
200 static pthread_t malloc_initializer = NO_INITIALIZER;
201 #else
202 # define NO_INITIALIZER false
203 # define INITIALIZER true
204 # define IS_INITIALIZER malloc_initializer
205 static bool malloc_initializer = NO_INITIALIZER;
206 #endif
207
208 /* Used to avoid initialization races. */
209 #ifdef _WIN32
210 #if _WIN32_WINNT >= 0x0600
211 static malloc_mutex_t init_lock = SRWLOCK_INIT;
212 #else
213 static malloc_mutex_t init_lock;
214 static bool init_lock_initialized = false;
215
JEMALLOC_ATTR(constructor)216 JEMALLOC_ATTR(constructor)
217 static void WINAPI
218 _init_init_lock(void)
219 {
220
221 /* If another constructor in the same binary is using mallctl to
222 * e.g. setup chunk hooks, it may end up running before this one,
223 * and malloc_init_hard will crash trying to lock the uninitialized
224 * lock. So we force an initialization of the lock in
225 * malloc_init_hard as well. We don't try to care about atomicity
226 * of the accessed to the init_lock_initialized boolean, since it
227 * really only matters early in the process creation, before any
228 * separate thread normally starts doing anything. */
229 if (!init_lock_initialized)
230 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT);
231 init_lock_initialized = true;
232 }
233
234 #ifdef _MSC_VER
235 # pragma section(".CRT$XCU", read)
236 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
237 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
238 #endif
239 #endif
240 #else
241 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
242 #endif
243
244 typedef struct {
245 void *p; /* Input pointer (as in realloc(p, s)). */
246 size_t s; /* Request size. */
247 void *r; /* Result pointer. */
248 } malloc_utrace_t;
249
250 #ifdef JEMALLOC_UTRACE
251 # define UTRACE(a, b, c) do { \
252 if (unlikely(opt_utrace)) { \
253 int utrace_serrno = errno; \
254 malloc_utrace_t ut; \
255 ut.p = (a); \
256 ut.s = (b); \
257 ut.r = (c); \
258 utrace(&ut, sizeof(ut)); \
259 errno = utrace_serrno; \
260 } \
261 } while (0)
262 #else
263 # define UTRACE(a, b, c)
264 #endif
265
266 /******************************************************************************/
267 /*
268 * Function prototypes for static functions that are referenced prior to
269 * definition.
270 */
271
272 static bool malloc_init_hard_a0(void);
273 static bool malloc_init_hard(void);
274
275 /******************************************************************************/
276 /*
277 * Begin miscellaneous support functions.
278 */
279
280 JEMALLOC_ALWAYS_INLINE_C bool
malloc_initialized(void)281 malloc_initialized(void)
282 {
283
284 return (malloc_init_state == malloc_init_initialized);
285 }
286
287 JEMALLOC_ALWAYS_INLINE_C void
malloc_thread_init(void)288 malloc_thread_init(void)
289 {
290
291 /*
292 * TSD initialization can't be safely done as a side effect of
293 * deallocation, because it is possible for a thread to do nothing but
294 * deallocate its TLS data via free(), in which case writing to TLS
295 * would cause write-after-free memory corruption. The quarantine
296 * facility *only* gets used as a side effect of deallocation, so make
297 * a best effort attempt at initializing its TSD by hooking all
298 * allocation events.
299 */
300 if (config_fill && unlikely(opt_quarantine))
301 quarantine_alloc_hook();
302 }
303
304 JEMALLOC_ALWAYS_INLINE_C bool
malloc_init_a0(void)305 malloc_init_a0(void)
306 {
307
308 if (unlikely(malloc_init_state == malloc_init_uninitialized))
309 return (malloc_init_hard_a0());
310 return (false);
311 }
312
313 JEMALLOC_ALWAYS_INLINE_C bool
malloc_init(void)314 malloc_init(void)
315 {
316
317 if (unlikely(!malloc_initialized()) && malloc_init_hard())
318 return (true);
319 malloc_thread_init();
320
321 return (false);
322 }
323
324 /*
325 * The a0*() functions are used instead of i{d,}alloc() in situations that
326 * cannot tolerate TLS variable access.
327 */
328
329 static void *
a0ialloc(size_t size,bool zero,bool is_metadata)330 a0ialloc(size_t size, bool zero, bool is_metadata)
331 {
332
333 if (unlikely(malloc_init_a0()))
334 return (NULL);
335
336 return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL,
337 is_metadata, arena_get(TSDN_NULL, 0, true), true));
338 }
339
340 static void
a0idalloc(void * ptr,bool is_metadata)341 a0idalloc(void *ptr, bool is_metadata)
342 {
343
344 idalloctm(TSDN_NULL, ptr, false, is_metadata, true);
345 }
346
347 arena_t *
a0get(void)348 a0get(void)
349 {
350
351 return (a0);
352 }
353
354 void *
a0malloc(size_t size)355 a0malloc(size_t size)
356 {
357
358 return (a0ialloc(size, false, true));
359 }
360
361 void
a0dalloc(void * ptr)362 a0dalloc(void *ptr)
363 {
364
365 a0idalloc(ptr, true);
366 }
367
368 /*
369 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
370 * situations that cannot tolerate TLS variable access (TLS allocation and very
371 * early internal data structure initialization).
372 */
373
374 void *
bootstrap_malloc(size_t size)375 bootstrap_malloc(size_t size)
376 {
377
378 if (unlikely(size == 0))
379 size = 1;
380
381 return (a0ialloc(size, false, false));
382 }
383
384 void *
bootstrap_calloc(size_t num,size_t size)385 bootstrap_calloc(size_t num, size_t size)
386 {
387 size_t num_size;
388
389 num_size = num * size;
390 if (unlikely(num_size == 0)) {
391 assert(num == 0 || size == 0);
392 num_size = 1;
393 }
394
395 return (a0ialloc(num_size, true, false));
396 }
397
398 void
bootstrap_free(void * ptr)399 bootstrap_free(void *ptr)
400 {
401
402 if (unlikely(ptr == NULL))
403 return;
404
405 a0idalloc(ptr, false);
406 }
407
408 static void
arena_set(unsigned ind,arena_t * arena)409 arena_set(unsigned ind, arena_t *arena)
410 {
411
412 atomic_write_p((void **)&arenas[ind], arena);
413 }
414
415 static void
narenas_total_set(unsigned narenas)416 narenas_total_set(unsigned narenas)
417 {
418
419 atomic_write_u(&narenas_total, narenas);
420 }
421
422 static void
narenas_total_inc(void)423 narenas_total_inc(void)
424 {
425
426 atomic_add_u(&narenas_total, 1);
427 }
428
429 unsigned
narenas_total_get(void)430 narenas_total_get(void)
431 {
432
433 return (atomic_read_u(&narenas_total));
434 }
435
436 /* Create a new arena and insert it into the arenas array at index ind. */
437 static arena_t *
arena_init_locked(tsdn_t * tsdn,unsigned ind)438 arena_init_locked(tsdn_t *tsdn, unsigned ind)
439 {
440 arena_t *arena;
441
442 assert(ind <= narenas_total_get());
443 if (ind > MALLOCX_ARENA_MAX)
444 return (NULL);
445 if (ind == narenas_total_get())
446 narenas_total_inc();
447
448 /*
449 * Another thread may have already initialized arenas[ind] if it's an
450 * auto arena.
451 */
452 arena = arena_get(tsdn, ind, false);
453 if (arena != NULL) {
454 assert(ind < narenas_auto);
455 return (arena);
456 }
457
458 /* Actually initialize the arena. */
459 arena = arena_new(tsdn, ind);
460 arena_set(ind, arena);
461 return (arena);
462 }
463
464 arena_t *
arena_init(tsdn_t * tsdn,unsigned ind)465 arena_init(tsdn_t *tsdn, unsigned ind)
466 {
467 arena_t *arena;
468
469 malloc_mutex_lock(tsdn, &arenas_lock);
470 arena = arena_init_locked(tsdn, ind);
471 malloc_mutex_unlock(tsdn, &arenas_lock);
472 return (arena);
473 }
474
475 static void
arena_bind(tsd_t * tsd,unsigned ind,bool internal)476 arena_bind(tsd_t *tsd, unsigned ind, bool internal)
477 {
478 arena_t *arena;
479
480 if (!tsd_nominal(tsd))
481 return;
482
483 arena = arena_get(tsd_tsdn(tsd), ind, false);
484 arena_nthreads_inc(arena, internal);
485
486 if (internal)
487 tsd_iarena_set(tsd, arena);
488 else
489 tsd_arena_set(tsd, arena);
490 }
491
492 void
arena_migrate(tsd_t * tsd,unsigned oldind,unsigned newind)493 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind)
494 {
495 arena_t *oldarena, *newarena;
496
497 oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
498 newarena = arena_get(tsd_tsdn(tsd), newind, false);
499 arena_nthreads_dec(oldarena, false);
500 arena_nthreads_inc(newarena, false);
501 tsd_arena_set(tsd, newarena);
502 }
503
504 static void
arena_unbind(tsd_t * tsd,unsigned ind,bool internal)505 arena_unbind(tsd_t *tsd, unsigned ind, bool internal)
506 {
507 arena_t *arena;
508
509 arena = arena_get(tsd_tsdn(tsd), ind, false);
510 arena_nthreads_dec(arena, internal);
511 if (internal)
512 tsd_iarena_set(tsd, NULL);
513 else
514 tsd_arena_set(tsd, NULL);
515 }
516
517 arena_tdata_t *
arena_tdata_get_hard(tsd_t * tsd,unsigned ind)518 arena_tdata_get_hard(tsd_t *tsd, unsigned ind)
519 {
520 arena_tdata_t *tdata, *arenas_tdata_old;
521 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
522 unsigned narenas_tdata_old, i;
523 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
524 unsigned narenas_actual = narenas_total_get();
525
526 /*
527 * Dissociate old tdata array (and set up for deallocation upon return)
528 * if it's too small.
529 */
530 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
531 arenas_tdata_old = arenas_tdata;
532 narenas_tdata_old = narenas_tdata;
533 arenas_tdata = NULL;
534 narenas_tdata = 0;
535 tsd_arenas_tdata_set(tsd, arenas_tdata);
536 tsd_narenas_tdata_set(tsd, narenas_tdata);
537 } else {
538 arenas_tdata_old = NULL;
539 narenas_tdata_old = 0;
540 }
541
542 /* Allocate tdata array if it's missing. */
543 if (arenas_tdata == NULL) {
544 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
545 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
546
547 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
548 *arenas_tdata_bypassp = true;
549 arenas_tdata = (arena_tdata_t *)a0malloc(
550 sizeof(arena_tdata_t) * narenas_tdata);
551 *arenas_tdata_bypassp = false;
552 }
553 if (arenas_tdata == NULL) {
554 tdata = NULL;
555 goto label_return;
556 }
557 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
558 tsd_arenas_tdata_set(tsd, arenas_tdata);
559 tsd_narenas_tdata_set(tsd, narenas_tdata);
560 }
561
562 /*
563 * Copy to tdata array. It's possible that the actual number of arenas
564 * has increased since narenas_total_get() was called above, but that
565 * causes no correctness issues unless two threads concurrently execute
566 * the arenas.extend mallctl, which we trust mallctl synchronization to
567 * prevent.
568 */
569
570 /* Copy/initialize tickers. */
571 for (i = 0; i < narenas_actual; i++) {
572 if (i < narenas_tdata_old) {
573 ticker_copy(&arenas_tdata[i].decay_ticker,
574 &arenas_tdata_old[i].decay_ticker);
575 } else {
576 ticker_init(&arenas_tdata[i].decay_ticker,
577 DECAY_NTICKS_PER_UPDATE);
578 }
579 }
580 if (narenas_tdata > narenas_actual) {
581 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
582 * (narenas_tdata - narenas_actual));
583 }
584
585 /* Read the refreshed tdata array. */
586 tdata = &arenas_tdata[ind];
587 label_return:
588 if (arenas_tdata_old != NULL)
589 a0dalloc(arenas_tdata_old);
590 return (tdata);
591 }
592
593 /* Slow path, called only by arena_choose(). */
594 arena_t *
arena_choose_hard(tsd_t * tsd,bool internal)595 arena_choose_hard(tsd_t *tsd, bool internal)
596 {
597 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
598
599 if (narenas_auto > 1) {
600 unsigned i, j, choose[2], first_null;
601
602 /*
603 * Determine binding for both non-internal and internal
604 * allocation.
605 *
606 * choose[0]: For application allocation.
607 * choose[1]: For internal metadata allocation.
608 */
609
610 for (j = 0; j < 2; j++)
611 choose[j] = 0;
612
613 first_null = narenas_auto;
614 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
615 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
616 for (i = 1; i < narenas_auto; i++) {
617 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
618 /*
619 * Choose the first arena that has the lowest
620 * number of threads assigned to it.
621 */
622 for (j = 0; j < 2; j++) {
623 if (arena_nthreads_get(arena_get(
624 tsd_tsdn(tsd), i, false), !!j) <
625 arena_nthreads_get(arena_get(
626 tsd_tsdn(tsd), choose[j], false),
627 !!j))
628 choose[j] = i;
629 }
630 } else if (first_null == narenas_auto) {
631 /*
632 * Record the index of the first uninitialized
633 * arena, in case all extant arenas are in use.
634 *
635 * NB: It is possible for there to be
636 * discontinuities in terms of initialized
637 * versus uninitialized arenas, due to the
638 * "thread.arena" mallctl.
639 */
640 first_null = i;
641 }
642 }
643
644 for (j = 0; j < 2; j++) {
645 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
646 choose[j], false), !!j) == 0 || first_null ==
647 narenas_auto) {
648 /*
649 * Use an unloaded arena, or the least loaded
650 * arena if all arenas are already initialized.
651 */
652 if (!!j == internal) {
653 ret = arena_get(tsd_tsdn(tsd),
654 choose[j], false);
655 }
656 } else {
657 arena_t *arena;
658
659 /* Initialize a new arena. */
660 choose[j] = first_null;
661 arena = arena_init_locked(tsd_tsdn(tsd),
662 choose[j]);
663 if (arena == NULL) {
664 malloc_mutex_unlock(tsd_tsdn(tsd),
665 &arenas_lock);
666 return (NULL);
667 }
668 if (!!j == internal)
669 ret = arena;
670 }
671 arena_bind(tsd, choose[j], !!j);
672 }
673 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
674 } else {
675 ret = arena_get(tsd_tsdn(tsd), 0, false);
676 arena_bind(tsd, 0, false);
677 arena_bind(tsd, 0, true);
678 }
679
680 return (ret);
681 }
682
683 void
thread_allocated_cleanup(tsd_t * tsd)684 thread_allocated_cleanup(tsd_t *tsd)
685 {
686
687 /* Do nothing. */
688 }
689
690 void
thread_deallocated_cleanup(tsd_t * tsd)691 thread_deallocated_cleanup(tsd_t *tsd)
692 {
693
694 /* Do nothing. */
695 }
696
697 void
iarena_cleanup(tsd_t * tsd)698 iarena_cleanup(tsd_t *tsd)
699 {
700 arena_t *iarena;
701
702 iarena = tsd_iarena_get(tsd);
703 if (iarena != NULL)
704 arena_unbind(tsd, iarena->ind, true);
705 }
706
707 void
arena_cleanup(tsd_t * tsd)708 arena_cleanup(tsd_t *tsd)
709 {
710 arena_t *arena;
711
712 arena = tsd_arena_get(tsd);
713 if (arena != NULL)
714 arena_unbind(tsd, arena->ind, false);
715 }
716
717 void
arenas_tdata_cleanup(tsd_t * tsd)718 arenas_tdata_cleanup(tsd_t *tsd)
719 {
720 arena_tdata_t *arenas_tdata;
721
722 /* Prevent tsd->arenas_tdata from being (re)created. */
723 *tsd_arenas_tdata_bypassp_get(tsd) = true;
724
725 arenas_tdata = tsd_arenas_tdata_get(tsd);
726 if (arenas_tdata != NULL) {
727 tsd_arenas_tdata_set(tsd, NULL);
728 a0dalloc(arenas_tdata);
729 }
730 }
731
732 void
narenas_tdata_cleanup(tsd_t * tsd)733 narenas_tdata_cleanup(tsd_t *tsd)
734 {
735
736 /* Do nothing. */
737 }
738
739 void
arenas_tdata_bypass_cleanup(tsd_t * tsd)740 arenas_tdata_bypass_cleanup(tsd_t *tsd)
741 {
742
743 /* Do nothing. */
744 }
745
746 static void
stats_print_atexit(void)747 stats_print_atexit(void)
748 {
749
750 if (config_tcache && config_stats) {
751 tsdn_t *tsdn;
752 unsigned narenas, i;
753
754 tsdn = tsdn_fetch();
755
756 /*
757 * Merge stats from extant threads. This is racy, since
758 * individual threads do not lock when recording tcache stats
759 * events. As a consequence, the final stats may be slightly
760 * out of date by the time they are reported, if other threads
761 * continue to allocate.
762 */
763 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
764 arena_t *arena = arena_get(tsdn, i, false);
765 if (arena != NULL) {
766 tcache_t *tcache;
767
768 /*
769 * tcache_stats_merge() locks bins, so if any
770 * code is introduced that acquires both arena
771 * and bin locks in the opposite order,
772 * deadlocks may result.
773 */
774 malloc_mutex_lock(tsdn, &arena->lock);
775 ql_foreach(tcache, &arena->tcache_ql, link) {
776 tcache_stats_merge(tsdn, tcache, arena);
777 }
778 malloc_mutex_unlock(tsdn, &arena->lock);
779 }
780 }
781 }
782 je_malloc_stats_print(NULL, NULL, NULL);
783 }
784
785 /*
786 * End miscellaneous support functions.
787 */
788 /******************************************************************************/
789 /*
790 * Begin initialization functions.
791 */
792
793 #ifndef JEMALLOC_HAVE_SECURE_GETENV
794 static char *
secure_getenv(const char * name)795 secure_getenv(const char *name)
796 {
797
798 # ifdef JEMALLOC_HAVE_ISSETUGID
799 if (issetugid() != 0)
800 return (NULL);
801 # endif
802 return (getenv(name));
803 }
804 #endif
805
806 static unsigned
malloc_ncpus(void)807 malloc_ncpus(void)
808 {
809 long result;
810
811 #ifdef _WIN32
812 SYSTEM_INFO si;
813 GetSystemInfo(&si);
814 result = si.dwNumberOfProcessors;
815 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
816 /*
817 * glibc >= 2.6 has the CPU_COUNT macro.
818 *
819 * glibc's sysconf() uses isspace(). glibc allocates for the first time
820 * *before* setting up the isspace tables. Therefore we need a
821 * different method to get the number of CPUs.
822 */
823 {
824 cpu_set_t set;
825
826 pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
827 result = CPU_COUNT(&set);
828 }
829 #else
830 result = sysconf(_SC_NPROCESSORS_ONLN);
831 #endif
832 return ((result == -1) ? 1 : (unsigned)result);
833 }
834
835 static bool
malloc_conf_next(char const ** opts_p,char const ** k_p,size_t * klen_p,char const ** v_p,size_t * vlen_p)836 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
837 char const **v_p, size_t *vlen_p)
838 {
839 bool accept;
840 const char *opts = *opts_p;
841
842 *k_p = opts;
843
844 for (accept = false; !accept;) {
845 switch (*opts) {
846 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
847 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
848 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
849 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
850 case 'Y': case 'Z':
851 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
852 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
853 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
854 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
855 case 'y': case 'z':
856 case '0': case '1': case '2': case '3': case '4': case '5':
857 case '6': case '7': case '8': case '9':
858 case '_':
859 opts++;
860 break;
861 case ':':
862 opts++;
863 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
864 *v_p = opts;
865 accept = true;
866 break;
867 case '\0':
868 if (opts != *opts_p) {
869 malloc_write("<jemalloc>: Conf string ends "
870 "with key\n");
871 }
872 return (true);
873 default:
874 malloc_write("<jemalloc>: Malformed conf string\n");
875 return (true);
876 }
877 }
878
879 for (accept = false; !accept;) {
880 switch (*opts) {
881 case ',':
882 opts++;
883 /*
884 * Look ahead one character here, because the next time
885 * this function is called, it will assume that end of
886 * input has been cleanly reached if no input remains,
887 * but we have optimistically already consumed the
888 * comma if one exists.
889 */
890 if (*opts == '\0') {
891 malloc_write("<jemalloc>: Conf string ends "
892 "with comma\n");
893 }
894 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
895 accept = true;
896 break;
897 case '\0':
898 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
899 accept = true;
900 break;
901 default:
902 opts++;
903 break;
904 }
905 }
906
907 *opts_p = opts;
908 return (false);
909 }
910
911 static void
malloc_conf_error(const char * msg,const char * k,size_t klen,const char * v,size_t vlen)912 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
913 size_t vlen)
914 {
915
916 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
917 (int)vlen, v);
918 }
919
920 static void
malloc_slow_flag_init(void)921 malloc_slow_flag_init(void)
922 {
923 /*
924 * Combine the runtime options into malloc_slow for fast path. Called
925 * after processing all the options.
926 */
927 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
928 | (opt_junk_free ? flag_opt_junk_free : 0)
929 | (opt_quarantine ? flag_opt_quarantine : 0)
930 | (opt_zero ? flag_opt_zero : 0)
931 | (opt_utrace ? flag_opt_utrace : 0)
932 | (opt_xmalloc ? flag_opt_xmalloc : 0);
933
934 if (config_valgrind)
935 malloc_slow_flags |= (in_valgrind ? flag_in_valgrind : 0);
936
937 malloc_slow = (malloc_slow_flags != 0);
938 }
939
940 static void
malloc_conf_init(void)941 malloc_conf_init(void)
942 {
943 unsigned i;
944 char buf[PATH_MAX + 1];
945 const char *opts, *k, *v;
946 size_t klen, vlen;
947
948 /*
949 * Automatically configure valgrind before processing options. The
950 * valgrind option remains in jemalloc 3.x for compatibility reasons.
951 */
952 if (config_valgrind) {
953 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
954 if (config_fill && unlikely(in_valgrind)) {
955 opt_junk = "false";
956 opt_junk_alloc = false;
957 opt_junk_free = false;
958 assert(!opt_zero);
959 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
960 opt_redzone = true;
961 }
962 if (config_tcache && unlikely(in_valgrind))
963 opt_tcache = false;
964 }
965
966 #if defined(__ANDROID__)
967 for (i = 0; i < 2; i++) {
968 #else
969 for (i = 0; i < 4; i++) {
970 #endif
971 /* Get runtime configuration. */
972 switch (i) {
973 case 0:
974 opts = config_malloc_conf;
975 break;
976 case 1:
977 if (je_malloc_conf != NULL) {
978 /*
979 * Use options that were compiled into the
980 * program.
981 */
982 opts = je_malloc_conf;
983 } else {
984 /* No configuration specified. */
985 buf[0] = '\0';
986 opts = buf;
987 }
988 break;
989 case 2: {
990 ssize_t linklen = 0;
991 #ifndef _WIN32
992 int saved_errno = errno;
993 const char *linkname =
994 # ifdef JEMALLOC_PREFIX
995 "/etc/"JEMALLOC_PREFIX"malloc.conf"
996 # else
997 "/etc/malloc.conf"
998 # endif
999 ;
1000
1001 /*
1002 * Try to use the contents of the "/etc/malloc.conf"
1003 * symbolic link's name.
1004 */
1005 linklen = readlink(linkname, buf, sizeof(buf) - 1);
1006 if (linklen == -1) {
1007 /* No configuration specified. */
1008 linklen = 0;
1009 /* Restore errno. */
1010 set_errno(saved_errno);
1011 }
1012 #endif
1013 buf[linklen] = '\0';
1014 opts = buf;
1015 break;
1016 } case 3: {
1017 const char *envname =
1018 #ifdef JEMALLOC_PREFIX
1019 JEMALLOC_CPREFIX"MALLOC_CONF"
1020 #else
1021 "MALLOC_CONF"
1022 #endif
1023 ;
1024
1025 if ((opts = secure_getenv(envname)) != NULL) {
1026 /*
1027 * Do nothing; opts is already initialized to
1028 * the value of the MALLOC_CONF environment
1029 * variable.
1030 */
1031 } else {
1032 /* No configuration specified. */
1033 buf[0] = '\0';
1034 opts = buf;
1035 }
1036 break;
1037 } default:
1038 not_reached();
1039 buf[0] = '\0';
1040 opts = buf;
1041 }
1042
1043 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
1044 &vlen)) {
1045 #define CONF_MATCH(n) \
1046 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
1047 #define CONF_MATCH_VALUE(n) \
1048 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
1049 #define CONF_HANDLE_BOOL(o, n, cont) \
1050 if (CONF_MATCH(n)) { \
1051 if (CONF_MATCH_VALUE("true")) \
1052 o = true; \
1053 else if (CONF_MATCH_VALUE("false")) \
1054 o = false; \
1055 else { \
1056 malloc_conf_error( \
1057 "Invalid conf value", \
1058 k, klen, v, vlen); \
1059 } \
1060 if (cont) \
1061 continue; \
1062 }
1063 #define CONF_MIN_no(um, min) false
1064 #define CONF_MIN_yes(um, min) ((um) < (min))
1065 #define CONF_MAX_no(um, max) false
1066 #define CONF_MAX_yes(um, max) ((um) > (max))
1067 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
1068 if (CONF_MATCH(n)) { \
1069 uintmax_t um; \
1070 char *end; \
1071 \
1072 set_errno(0); \
1073 um = malloc_strtoumax(v, &end, 0); \
1074 if (get_errno() != 0 || (uintptr_t)end -\
1075 (uintptr_t)v != vlen) { \
1076 malloc_conf_error( \
1077 "Invalid conf value", \
1078 k, klen, v, vlen); \
1079 } else if (clip) { \
1080 if (CONF_MIN_##check_min(um, \
1081 (min))) \
1082 o = (t)(min); \
1083 else if (CONF_MAX_##check_max( \
1084 um, (max))) \
1085 o = (t)(max); \
1086 else \
1087 o = (t)um; \
1088 } else { \
1089 if (CONF_MIN_##check_min(um, \
1090 (min)) || \
1091 CONF_MAX_##check_max(um, \
1092 (max))) { \
1093 malloc_conf_error( \
1094 "Out-of-range " \
1095 "conf value", \
1096 k, klen, v, vlen); \
1097 } else \
1098 o = (t)um; \
1099 } \
1100 continue; \
1101 }
1102 #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \
1103 clip) \
1104 CONF_HANDLE_T_U(unsigned, o, n, min, max, \
1105 check_min, check_max, clip)
1106 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \
1107 CONF_HANDLE_T_U(size_t, o, n, min, max, \
1108 check_min, check_max, clip)
1109 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \
1110 if (CONF_MATCH(n)) { \
1111 long l; \
1112 char *end; \
1113 \
1114 set_errno(0); \
1115 l = strtol(v, &end, 0); \
1116 if (get_errno() != 0 || (uintptr_t)end -\
1117 (uintptr_t)v != vlen) { \
1118 malloc_conf_error( \
1119 "Invalid conf value", \
1120 k, klen, v, vlen); \
1121 } else if (l < (ssize_t)(min) || l > \
1122 (ssize_t)(max)) { \
1123 malloc_conf_error( \
1124 "Out-of-range conf value", \
1125 k, klen, v, vlen); \
1126 } else \
1127 o = l; \
1128 continue; \
1129 }
1130 #define CONF_HANDLE_CHAR_P(o, n, d) \
1131 if (CONF_MATCH(n)) { \
1132 size_t cpylen = (vlen <= \
1133 sizeof(o)-1) ? vlen : \
1134 sizeof(o)-1; \
1135 strncpy(o, v, cpylen); \
1136 o[cpylen] = '\0'; \
1137 continue; \
1138 }
1139
1140 CONF_HANDLE_BOOL(opt_abort, "abort", true)
1141 /*
1142 * Chunks always require at least one header page,
1143 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and
1144 * possibly an additional page in the presence of
1145 * redzones. In order to simplify options processing,
1146 * use a conservative bound that accommodates all these
1147 * constraints.
1148 */
1149 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
1150 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1),
1151 (sizeof(size_t) << 3) - 1, yes, yes, true)
1152 if (strncmp("dss", k, klen) == 0) {
1153 int i;
1154 bool match = false;
1155 for (i = 0; i < dss_prec_limit; i++) {
1156 if (strncmp(dss_prec_names[i], v, vlen)
1157 == 0) {
1158 if (chunk_dss_prec_set(i)) {
1159 malloc_conf_error(
1160 "Error setting dss",
1161 k, klen, v, vlen);
1162 } else {
1163 opt_dss =
1164 dss_prec_names[i];
1165 match = true;
1166 break;
1167 }
1168 }
1169 }
1170 if (!match) {
1171 malloc_conf_error("Invalid conf value",
1172 k, klen, v, vlen);
1173 }
1174 continue;
1175 }
1176 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
1177 UINT_MAX, yes, no, false)
1178 if (strncmp("purge", k, klen) == 0) {
1179 int i;
1180 bool match = false;
1181 for (i = 0; i < purge_mode_limit; i++) {
1182 if (strncmp(purge_mode_names[i], v,
1183 vlen) == 0) {
1184 opt_purge = (purge_mode_t)i;
1185 match = true;
1186 break;
1187 }
1188 }
1189 if (!match) {
1190 malloc_conf_error("Invalid conf value",
1191 k, klen, v, vlen);
1192 }
1193 continue;
1194 }
1195 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
1196 -1, (sizeof(size_t) << 3) - 1)
1197 CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1,
1198 NSTIME_SEC_MAX);
1199 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
1200 if (config_fill) {
1201 if (CONF_MATCH("junk")) {
1202 if (CONF_MATCH_VALUE("true")) {
1203 if (config_valgrind &&
1204 unlikely(in_valgrind)) {
1205 malloc_conf_error(
1206 "Deallocation-time "
1207 "junk filling cannot "
1208 "be enabled while "
1209 "running inside "
1210 "Valgrind", k, klen, v,
1211 vlen);
1212 } else {
1213 opt_junk = "true";
1214 opt_junk_alloc = true;
1215 opt_junk_free = true;
1216 }
1217 } else if (CONF_MATCH_VALUE("false")) {
1218 opt_junk = "false";
1219 opt_junk_alloc = opt_junk_free =
1220 false;
1221 } else if (CONF_MATCH_VALUE("alloc")) {
1222 opt_junk = "alloc";
1223 opt_junk_alloc = true;
1224 opt_junk_free = false;
1225 } else if (CONF_MATCH_VALUE("free")) {
1226 if (config_valgrind &&
1227 unlikely(in_valgrind)) {
1228 malloc_conf_error(
1229 "Deallocation-time "
1230 "junk filling cannot "
1231 "be enabled while "
1232 "running inside "
1233 "Valgrind", k, klen, v,
1234 vlen);
1235 } else {
1236 opt_junk = "free";
1237 opt_junk_alloc = false;
1238 opt_junk_free = true;
1239 }
1240 } else {
1241 malloc_conf_error(
1242 "Invalid conf value", k,
1243 klen, v, vlen);
1244 }
1245 continue;
1246 }
1247 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
1248 0, SIZE_T_MAX, no, no, false)
1249 CONF_HANDLE_BOOL(opt_redzone, "redzone", true)
1250 CONF_HANDLE_BOOL(opt_zero, "zero", true)
1251 }
1252 if (config_utrace) {
1253 CONF_HANDLE_BOOL(opt_utrace, "utrace", true)
1254 }
1255 if (config_xmalloc) {
1256 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true)
1257 }
1258 if (config_tcache) {
1259 CONF_HANDLE_BOOL(opt_tcache, "tcache",
1260 !config_valgrind || !in_valgrind)
1261 if (CONF_MATCH("tcache")) {
1262 assert(config_valgrind && in_valgrind);
1263 if (opt_tcache) {
1264 opt_tcache = false;
1265 malloc_conf_error(
1266 "tcache cannot be enabled "
1267 "while running inside Valgrind",
1268 k, klen, v, vlen);
1269 }
1270 continue;
1271 }
1272 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
1273 "lg_tcache_max", -1,
1274 (sizeof(size_t) << 3) - 1)
1275 }
1276 if (config_prof) {
1277 CONF_HANDLE_BOOL(opt_prof, "prof", true)
1278 CONF_HANDLE_CHAR_P(opt_prof_prefix,
1279 "prof_prefix", "jeprof")
1280 CONF_HANDLE_BOOL(opt_prof_active, "prof_active",
1281 true)
1282 CONF_HANDLE_BOOL(opt_prof_thread_active_init,
1283 "prof_thread_active_init", true)
1284 CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
1285 "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
1286 - 1, no, yes, true)
1287 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
1288 true)
1289 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
1290 "lg_prof_interval", -1,
1291 (sizeof(uint64_t) << 3) - 1)
1292 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump",
1293 true)
1294 CONF_HANDLE_BOOL(opt_prof_final, "prof_final",
1295 true)
1296 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak",
1297 true)
1298 }
1299 malloc_conf_error("Invalid conf pair", k, klen, v,
1300 vlen);
1301 #undef CONF_MATCH
1302 #undef CONF_MATCH_VALUE
1303 #undef CONF_HANDLE_BOOL
1304 #undef CONF_MIN_no
1305 #undef CONF_MIN_yes
1306 #undef CONF_MAX_no
1307 #undef CONF_MAX_yes
1308 #undef CONF_HANDLE_T_U
1309 #undef CONF_HANDLE_UNSIGNED
1310 #undef CONF_HANDLE_SIZE_T
1311 #undef CONF_HANDLE_SSIZE_T
1312 #undef CONF_HANDLE_CHAR_P
1313 }
1314 }
1315 }
1316
1317 static bool
1318 malloc_init_hard_needed(void)
1319 {
1320
1321 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
1322 malloc_init_recursible)) {
1323 /*
1324 * Another thread initialized the allocator before this one
1325 * acquired init_lock, or this thread is the initializing
1326 * thread, and it is recursively allocating.
1327 */
1328 return (false);
1329 }
1330 #ifdef JEMALLOC_THREADED_INIT
1331 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
1332 spin_t spinner;
1333
1334 /* Busy-wait until the initializing thread completes. */
1335 spin_init(&spinner);
1336 do {
1337 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1338 spin_adaptive(&spinner);
1339 malloc_mutex_lock(TSDN_NULL, &init_lock);
1340 } while (!malloc_initialized());
1341 return (false);
1342 }
1343 #endif
1344 return (true);
1345 }
1346
1347 static bool
1348 malloc_init_hard_a0_locked()
1349 {
1350
1351 malloc_initializer = INITIALIZER;
1352
1353 if (config_prof)
1354 prof_boot0();
1355 malloc_conf_init();
1356 if (opt_stats_print) {
1357 /* Print statistics at exit. */
1358 if (atexit(stats_print_atexit) != 0) {
1359 malloc_write("<jemalloc>: Error in atexit()\n");
1360 if (opt_abort)
1361 abort();
1362 }
1363 }
1364 pages_boot();
1365 if (base_boot())
1366 return (true);
1367 if (chunk_boot())
1368 return (true);
1369 if (ctl_boot())
1370 return (true);
1371 if (config_prof)
1372 prof_boot1();
1373 arena_boot();
1374 if (config_tcache && tcache_boot(TSDN_NULL))
1375 return (true);
1376 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS))
1377 return (true);
1378 /*
1379 * Create enough scaffolding to allow recursive allocation in
1380 * malloc_ncpus().
1381 */
1382 narenas_auto = 1;
1383 narenas_total_set(narenas_auto);
1384 arenas = &a0;
1385 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
1386 /*
1387 * Initialize one arena here. The rest are lazily created in
1388 * arena_choose_hard().
1389 */
1390 if (arena_init(TSDN_NULL, 0) == NULL)
1391 return (true);
1392
1393 malloc_init_state = malloc_init_a0_initialized;
1394
1395 return (false);
1396 }
1397
1398 static bool
1399 malloc_init_hard_a0(void)
1400 {
1401 bool ret;
1402
1403 malloc_mutex_lock(TSDN_NULL, &init_lock);
1404 ret = malloc_init_hard_a0_locked();
1405 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1406 return (ret);
1407 }
1408
1409 /* Initialize data structures which may trigger recursive allocation. */
1410 static bool
1411 malloc_init_hard_recursible(void)
1412 {
1413
1414 malloc_init_state = malloc_init_recursible;
1415
1416 ncpus = malloc_ncpus();
1417
1418 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
1419 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
1420 !defined(__native_client__))
1421 /* LinuxThreads' pthread_atfork() allocates. */
1422 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
1423 jemalloc_postfork_child) != 0) {
1424 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
1425 if (opt_abort)
1426 abort();
1427 return (true);
1428 }
1429 #endif
1430
1431 return (false);
1432 }
1433
1434 static bool
1435 malloc_init_hard_finish(tsdn_t *tsdn)
1436 {
1437
1438 if (malloc_mutex_boot())
1439 return (true);
1440
1441 if (opt_narenas == 0) {
1442 /*
1443 * For SMP systems, create more than one arena per CPU by
1444 * default.
1445 */
1446 if (ncpus > 1)
1447 opt_narenas = ncpus << 2;
1448 else
1449 opt_narenas = 1;
1450 }
1451 #if defined(ANDROID_MAX_ARENAS)
1452 /* Never create more than MAX_ARENAS arenas regardless of num_cpus.
1453 * Extra arenas use more PSS and are not very useful unless
1454 * lots of threads are allocing/freeing at the same time.
1455 */
1456 if (opt_narenas > ANDROID_MAX_ARENAS)
1457 opt_narenas = ANDROID_MAX_ARENAS;
1458 #endif
1459 narenas_auto = opt_narenas;
1460 /*
1461 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
1462 */
1463 if (narenas_auto > MALLOCX_ARENA_MAX) {
1464 narenas_auto = MALLOCX_ARENA_MAX;
1465 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
1466 narenas_auto);
1467 }
1468 narenas_total_set(narenas_auto);
1469
1470 /* Allocate and initialize arenas. */
1471 arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) *
1472 (MALLOCX_ARENA_MAX+1));
1473 if (arenas == NULL)
1474 return (true);
1475 /* Copy the pointer to the one arena that was already initialized. */
1476 arena_set(0, a0);
1477
1478 malloc_init_state = malloc_init_initialized;
1479 malloc_slow_flag_init();
1480
1481 return (false);
1482 }
1483
1484 static bool
1485 malloc_init_hard(void)
1486 {
1487 tsd_t *tsd;
1488
1489 #if defined(_WIN32) && _WIN32_WINNT < 0x0600
1490 _init_init_lock();
1491 #endif
1492 malloc_mutex_lock(TSDN_NULL, &init_lock);
1493 if (!malloc_init_hard_needed()) {
1494 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1495 return (false);
1496 }
1497
1498 if (malloc_init_state != malloc_init_a0_initialized &&
1499 malloc_init_hard_a0_locked()) {
1500 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1501 return (true);
1502 }
1503
1504 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1505 /* Recursive allocation relies on functional tsd. */
1506 tsd = malloc_tsd_boot0();
1507 if (tsd == NULL)
1508 return (true);
1509 if (malloc_init_hard_recursible())
1510 return (true);
1511 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
1512
1513 if (config_prof && prof_boot2(tsd)) {
1514 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1515 return (true);
1516 }
1517
1518 if (malloc_init_hard_finish(tsd_tsdn(tsd))) {
1519 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1520 return (true);
1521 }
1522
1523 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1524 malloc_tsd_boot1();
1525 return (false);
1526 }
1527
1528 /*
1529 * End initialization functions.
1530 */
1531 /******************************************************************************/
1532 /*
1533 * Begin malloc(3)-compatible functions.
1534 */
1535
1536 static void *
1537 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
1538 prof_tctx_t *tctx, bool slow_path)
1539 {
1540 void *p;
1541
1542 if (tctx == NULL)
1543 return (NULL);
1544 if (usize <= SMALL_MAXCLASS) {
1545 szind_t ind_large = size2index(LARGE_MINCLASS);
1546 p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
1547 if (p == NULL)
1548 return (NULL);
1549 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1550 } else
1551 p = ialloc(tsd, usize, ind, zero, slow_path);
1552
1553 return (p);
1554 }
1555
1556 JEMALLOC_ALWAYS_INLINE_C void *
1557 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
1558 {
1559 void *p;
1560 prof_tctx_t *tctx;
1561
1562 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1563 if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
1564 p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path);
1565 else
1566 p = ialloc(tsd, usize, ind, zero, slow_path);
1567 if (unlikely(p == NULL)) {
1568 prof_alloc_rollback(tsd, tctx, true);
1569 return (NULL);
1570 }
1571 prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1572
1573 return (p);
1574 }
1575
1576 /*
1577 * ialloc_body() is inlined so that fast and slow paths are generated separately
1578 * with statically known slow_path.
1579 *
1580 * This function guarantees that *tsdn is non-NULL on success.
1581 */
1582 JEMALLOC_ALWAYS_INLINE_C void *
1583 ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
1584 bool slow_path)
1585 {
1586 tsd_t *tsd;
1587 szind_t ind;
1588
1589 if (slow_path && unlikely(malloc_init())) {
1590 *tsdn = NULL;
1591 return (NULL);
1592 }
1593
1594 tsd = tsd_fetch();
1595 *tsdn = tsd_tsdn(tsd);
1596 witness_assert_lockless(tsd_tsdn(tsd));
1597
1598 ind = size2index(size);
1599 if (unlikely(ind >= NSIZES))
1600 return (NULL);
1601
1602 if (config_stats || (config_prof && opt_prof) || (slow_path &&
1603 config_valgrind && unlikely(in_valgrind))) {
1604 *usize = index2size(ind);
1605 assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
1606 }
1607
1608 if (config_prof && opt_prof)
1609 return (ialloc_prof(tsd, *usize, ind, zero, slow_path));
1610
1611 return (ialloc(tsd, size, ind, zero, slow_path));
1612 }
1613
1614 JEMALLOC_ALWAYS_INLINE_C void
1615 ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
1616 bool update_errno, bool slow_path)
1617 {
1618
1619 assert(!tsdn_null(tsdn) || ret == NULL);
1620
1621 if (unlikely(ret == NULL)) {
1622 if (slow_path && config_xmalloc && unlikely(opt_xmalloc)) {
1623 malloc_printf("<jemalloc>: Error in %s(): out of "
1624 "memory\n", func);
1625 abort();
1626 }
1627 if (update_errno)
1628 set_errno(ENOMEM);
1629 }
1630 if (config_stats && likely(ret != NULL)) {
1631 assert(usize == isalloc(tsdn, ret, config_prof));
1632 *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
1633 }
1634 witness_assert_lockless(tsdn);
1635 }
1636
1637 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1638 void JEMALLOC_NOTHROW *
1639 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
1640 je_malloc(size_t size)
1641 {
1642 void *ret;
1643 tsdn_t *tsdn;
1644 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1645
1646 if (size == 0)
1647 size = 1;
1648
1649 if (likely(!malloc_slow)) {
1650 ret = ialloc_body(size, false, &tsdn, &usize, false);
1651 ialloc_post_check(ret, tsdn, usize, "malloc", true, false);
1652 } else {
1653 ret = ialloc_body(size, false, &tsdn, &usize, true);
1654 ialloc_post_check(ret, tsdn, usize, "malloc", true, true);
1655 UTRACE(0, size, ret);
1656 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false);
1657 }
1658
1659 return (ret);
1660 }
1661
1662 static void *
1663 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
1664 prof_tctx_t *tctx)
1665 {
1666 void *p;
1667
1668 if (tctx == NULL)
1669 return (NULL);
1670 if (usize <= SMALL_MAXCLASS) {
1671 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS);
1672 p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
1673 if (p == NULL)
1674 return (NULL);
1675 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1676 } else
1677 p = ipalloc(tsd, usize, alignment, false);
1678
1679 return (p);
1680 }
1681
1682 JEMALLOC_ALWAYS_INLINE_C void *
1683 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
1684 {
1685 void *p;
1686 prof_tctx_t *tctx;
1687
1688 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1689 if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
1690 p = imemalign_prof_sample(tsd, alignment, usize, tctx);
1691 else
1692 p = ipalloc(tsd, usize, alignment, false);
1693 if (unlikely(p == NULL)) {
1694 prof_alloc_rollback(tsd, tctx, true);
1695 return (NULL);
1696 }
1697 prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1698
1699 return (p);
1700 }
1701
1702 JEMALLOC_ATTR(nonnull(1))
1703 static int
1704 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
1705 {
1706 int ret;
1707 tsd_t *tsd;
1708 size_t usize;
1709 void *result;
1710
1711 assert(min_alignment != 0);
1712
1713 if (unlikely(malloc_init())) {
1714 tsd = NULL;
1715 result = NULL;
1716 goto label_oom;
1717 }
1718 tsd = tsd_fetch();
1719 witness_assert_lockless(tsd_tsdn(tsd));
1720 if (size == 0)
1721 size = 1;
1722
1723 /* Make sure that alignment is a large enough power of 2. */
1724 if (unlikely(((alignment - 1) & alignment) != 0
1725 || (alignment < min_alignment))) {
1726 if (config_xmalloc && unlikely(opt_xmalloc)) {
1727 malloc_write("<jemalloc>: Error allocating "
1728 "aligned memory: invalid alignment\n");
1729 abort();
1730 }
1731 result = NULL;
1732 ret = EINVAL;
1733 goto label_return;
1734 }
1735
1736 usize = sa2u(size, alignment);
1737 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
1738 result = NULL;
1739 goto label_oom;
1740 }
1741
1742 if (config_prof && opt_prof)
1743 result = imemalign_prof(tsd, alignment, usize);
1744 else
1745 result = ipalloc(tsd, usize, alignment, false);
1746 if (unlikely(result == NULL))
1747 goto label_oom;
1748 assert(((uintptr_t)result & (alignment - 1)) == ZU(0));
1749
1750 *memptr = result;
1751 ret = 0;
1752 label_return:
1753 if (config_stats && likely(result != NULL)) {
1754 assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
1755 *tsd_thread_allocatedp_get(tsd) += usize;
1756 }
1757 UTRACE(0, size, result);
1758 JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
1759 false);
1760 witness_assert_lockless(tsd_tsdn(tsd));
1761 return (ret);
1762 label_oom:
1763 assert(result == NULL);
1764 if (config_xmalloc && unlikely(opt_xmalloc)) {
1765 malloc_write("<jemalloc>: Error allocating aligned memory: "
1766 "out of memory\n");
1767 abort();
1768 }
1769 ret = ENOMEM;
1770 witness_assert_lockless(tsd_tsdn(tsd));
1771 goto label_return;
1772 }
1773
1774 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
1775 JEMALLOC_ATTR(nonnull(1))
1776 je_posix_memalign(void **memptr, size_t alignment, size_t size)
1777 {
1778 int ret;
1779
1780 ret = imemalign(memptr, alignment, size, sizeof(void *));
1781
1782 return (ret);
1783 }
1784
1785 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1786 void JEMALLOC_NOTHROW *
1787 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
1788 je_aligned_alloc(size_t alignment, size_t size)
1789 {
1790 void *ret;
1791 int err;
1792
1793 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) {
1794 ret = NULL;
1795 set_errno(err);
1796 }
1797
1798 return (ret);
1799 }
1800
1801 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1802 void JEMALLOC_NOTHROW *
1803 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
1804 je_calloc(size_t num, size_t size)
1805 {
1806 void *ret;
1807 tsdn_t *tsdn;
1808 size_t num_size;
1809 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1810
1811 num_size = num * size;
1812 if (unlikely(num_size == 0)) {
1813 if (num == 0 || size == 0)
1814 num_size = 1;
1815 else
1816 num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */
1817 /*
1818 * Try to avoid division here. We know that it isn't possible to
1819 * overflow during multiplication if neither operand uses any of the
1820 * most significant half of the bits in a size_t.
1821 */
1822 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) <<
1823 2))) && (num_size / size != num)))
1824 num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */
1825
1826 if (likely(!malloc_slow)) {
1827 ret = ialloc_body(num_size, true, &tsdn, &usize, false);
1828 ialloc_post_check(ret, tsdn, usize, "calloc", true, false);
1829 } else {
1830 ret = ialloc_body(num_size, true, &tsdn, &usize, true);
1831 ialloc_post_check(ret, tsdn, usize, "calloc", true, true);
1832 UTRACE(0, num_size, ret);
1833 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true);
1834 }
1835
1836 return (ret);
1837 }
1838
1839 static void *
1840 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
1841 prof_tctx_t *tctx)
1842 {
1843 void *p;
1844
1845 if (tctx == NULL)
1846 return (NULL);
1847 if (usize <= SMALL_MAXCLASS) {
1848 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
1849 if (p == NULL)
1850 return (NULL);
1851 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1852 } else
1853 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1854
1855 return (p);
1856 }
1857
1858 JEMALLOC_ALWAYS_INLINE_C void *
1859 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
1860 {
1861 void *p;
1862 bool prof_active;
1863 prof_tctx_t *old_tctx, *tctx;
1864
1865 prof_active = prof_active_get_unlocked();
1866 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
1867 tctx = prof_alloc_prep(tsd, usize, prof_active, true);
1868 if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
1869 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
1870 else
1871 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1872 if (unlikely(p == NULL)) {
1873 prof_alloc_rollback(tsd, tctx, true);
1874 return (NULL);
1875 }
1876 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
1877 old_tctx);
1878
1879 return (p);
1880 }
1881
1882 JEMALLOC_INLINE_C void
1883 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
1884 {
1885 size_t usize;
1886 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1887
1888 witness_assert_lockless(tsd_tsdn(tsd));
1889
1890 assert(ptr != NULL);
1891 assert(malloc_initialized() || IS_INITIALIZER);
1892
1893 if (config_prof && opt_prof) {
1894 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1895 prof_free(tsd, ptr, usize);
1896 } else if (config_stats || config_valgrind)
1897 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1898 if (config_stats)
1899 *tsd_thread_deallocatedp_get(tsd) += usize;
1900
1901 if (likely(!slow_path))
1902 iqalloc(tsd, ptr, tcache, false);
1903 else {
1904 if (config_valgrind && unlikely(in_valgrind))
1905 rzsize = p2rz(tsd_tsdn(tsd), ptr);
1906 iqalloc(tsd, ptr, tcache, true);
1907 JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1908 }
1909 }
1910
1911 JEMALLOC_INLINE_C void
1912 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
1913 {
1914 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1915
1916 witness_assert_lockless(tsd_tsdn(tsd));
1917
1918 assert(ptr != NULL);
1919 assert(malloc_initialized() || IS_INITIALIZER);
1920
1921 if (config_prof && opt_prof)
1922 prof_free(tsd, ptr, usize);
1923 if (config_stats)
1924 *tsd_thread_deallocatedp_get(tsd) += usize;
1925 if (config_valgrind && unlikely(in_valgrind))
1926 rzsize = p2rz(tsd_tsdn(tsd), ptr);
1927 isqalloc(tsd, ptr, usize, tcache, slow_path);
1928 JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1929 }
1930
1931 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
1932 void JEMALLOC_NOTHROW *
1933 JEMALLOC_ALLOC_SIZE(2)
1934 je_realloc(void *ptr, size_t size)
1935 {
1936 void *ret;
1937 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
1938 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1939 size_t old_usize = 0;
1940 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1941
1942 if (unlikely(size == 0)) {
1943 if (ptr != NULL) {
1944 tsd_t *tsd;
1945
1946 /* realloc(ptr, 0) is equivalent to free(ptr). */
1947 UTRACE(ptr, 0, 0);
1948 tsd = tsd_fetch();
1949 ifree(tsd, ptr, tcache_get(tsd, false), true);
1950 return (NULL);
1951 }
1952 size = 1;
1953 }
1954
1955 if (likely(ptr != NULL)) {
1956 tsd_t *tsd;
1957
1958 assert(malloc_initialized() || IS_INITIALIZER);
1959 malloc_thread_init();
1960 tsd = tsd_fetch();
1961
1962 witness_assert_lockless(tsd_tsdn(tsd));
1963
1964 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1965 if (config_valgrind && unlikely(in_valgrind)) {
1966 old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) :
1967 u2rz(old_usize);
1968 }
1969
1970 if (config_prof && opt_prof) {
1971 usize = s2u(size);
1972 ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
1973 NULL : irealloc_prof(tsd, ptr, old_usize, usize);
1974 } else {
1975 if (config_stats || (config_valgrind &&
1976 unlikely(in_valgrind)))
1977 usize = s2u(size);
1978 ret = iralloc(tsd, ptr, old_usize, size, 0, false);
1979 }
1980 tsdn = tsd_tsdn(tsd);
1981 } else {
1982 /* realloc(NULL, size) is equivalent to malloc(size). */
1983 if (likely(!malloc_slow))
1984 ret = ialloc_body(size, false, &tsdn, &usize, false);
1985 else
1986 ret = ialloc_body(size, false, &tsdn, &usize, true);
1987 assert(!tsdn_null(tsdn) || ret == NULL);
1988 }
1989
1990 if (unlikely(ret == NULL)) {
1991 if (config_xmalloc && unlikely(opt_xmalloc)) {
1992 malloc_write("<jemalloc>: Error in realloc(): "
1993 "out of memory\n");
1994 abort();
1995 }
1996 set_errno(ENOMEM);
1997 }
1998 if (config_stats && likely(ret != NULL)) {
1999 tsd_t *tsd;
2000
2001 assert(usize == isalloc(tsdn, ret, config_prof));
2002 tsd = tsdn_tsd(tsdn);
2003 *tsd_thread_allocatedp_get(tsd) += usize;
2004 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2005 }
2006 UTRACE(ptr, size, ret);
2007 JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr,
2008 old_usize, old_rzsize, maybe, false);
2009 witness_assert_lockless(tsdn);
2010 return (ret);
2011 }
2012
2013 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2014 je_free(void *ptr)
2015 {
2016
2017 UTRACE(ptr, 0, 0);
2018 if (likely(ptr != NULL)) {
2019 tsd_t *tsd = tsd_fetch();
2020 witness_assert_lockless(tsd_tsdn(tsd));
2021 if (likely(!malloc_slow))
2022 ifree(tsd, ptr, tcache_get(tsd, false), false);
2023 else
2024 ifree(tsd, ptr, tcache_get(tsd, false), true);
2025 witness_assert_lockless(tsd_tsdn(tsd));
2026 }
2027 }
2028
2029 /*
2030 * End malloc(3)-compatible functions.
2031 */
2032 /******************************************************************************/
2033 /*
2034 * Begin non-standard override functions.
2035 */
2036
2037 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
2038 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2039 void JEMALLOC_NOTHROW *
2040 JEMALLOC_ATTR(malloc)
2041 je_memalign(size_t alignment, size_t size)
2042 {
2043 void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
2044 if (unlikely(imemalign(&ret, alignment, size, 1) != 0))
2045 ret = NULL;
2046 return (ret);
2047 }
2048 #endif
2049
2050 #ifdef JEMALLOC_OVERRIDE_VALLOC
2051 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2052 void JEMALLOC_NOTHROW *
2053 JEMALLOC_ATTR(malloc)
2054 je_valloc(size_t size)
2055 {
2056 void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
2057 if (unlikely(imemalign(&ret, PAGE, size, 1) != 0))
2058 ret = NULL;
2059 return (ret);
2060 }
2061 #endif
2062
2063 /*
2064 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
2065 * #define je_malloc malloc
2066 */
2067 #define malloc_is_malloc 1
2068 #define is_malloc_(a) malloc_is_ ## a
2069 #define is_malloc(a) is_malloc_(a)
2070
2071 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK))
2072 /*
2073 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
2074 * to inconsistently reference libc's malloc(3)-compatible functions
2075 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
2076 *
2077 * These definitions interpose hooks in glibc. The functions are actually
2078 * passed an extra argument for the caller return address, which will be
2079 * ignored.
2080 */
2081 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
2082 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
2083 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
2084 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
2085 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
2086 je_memalign;
2087 # endif
2088
2089 #ifdef CPU_COUNT
2090 /*
2091 * To enable static linking with glibc, the libc specific malloc interface must
2092 * be implemented also, so none of glibc's malloc.o functions are added to the
2093 * link.
2094 */
2095 #define ALIAS(je_fn) __attribute__((alias (#je_fn), used))
2096 /* To force macro expansion of je_ prefix before stringification. */
2097 #define PREALIAS(je_fn) ALIAS(je_fn)
2098 void *__libc_malloc(size_t size) PREALIAS(je_malloc);
2099 void __libc_free(void* ptr) PREALIAS(je_free);
2100 void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
2101 void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
2102 void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
2103 void *__libc_valloc(size_t size) PREALIAS(je_valloc);
2104 int __posix_memalign(void** r, size_t a, size_t s)
2105 PREALIAS(je_posix_memalign);
2106 #undef PREALIAS
2107 #undef ALIAS
2108
2109 #endif
2110
2111 #endif
2112
2113 /*
2114 * End non-standard override functions.
2115 */
2116 /******************************************************************************/
2117 /*
2118 * Begin non-standard functions.
2119 */
2120
2121 JEMALLOC_ALWAYS_INLINE_C bool
2122 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,
2123 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena)
2124 {
2125
2126 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) {
2127 *alignment = 0;
2128 *usize = s2u(size);
2129 } else {
2130 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
2131 *usize = sa2u(size, *alignment);
2132 }
2133 if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS))
2134 return (true);
2135 *zero = MALLOCX_ZERO_GET(flags);
2136 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
2137 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2138 *tcache = NULL;
2139 else
2140 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2141 } else
2142 *tcache = tcache_get(tsd, true);
2143 if ((flags & MALLOCX_ARENA_MASK) != 0) {
2144 unsigned arena_ind = MALLOCX_ARENA_GET(flags);
2145 *arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
2146 if (unlikely(*arena == NULL))
2147 return (true);
2148 } else
2149 *arena = NULL;
2150 return (false);
2151 }
2152
2153 JEMALLOC_ALWAYS_INLINE_C void *
2154 imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
2155 tcache_t *tcache, arena_t *arena, bool slow_path)
2156 {
2157 szind_t ind;
2158
2159 if (unlikely(alignment != 0))
2160 return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
2161 ind = size2index(usize);
2162 assert(ind < NSIZES);
2163 return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena,
2164 slow_path));
2165 }
2166
2167 static void *
2168 imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
2169 tcache_t *tcache, arena_t *arena, bool slow_path)
2170 {
2171 void *p;
2172
2173 if (usize <= SMALL_MAXCLASS) {
2174 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) :
2175 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS);
2176 p = imallocx_flags(tsdn, LARGE_MINCLASS, alignment, zero,
2177 tcache, arena, slow_path);
2178 if (p == NULL)
2179 return (NULL);
2180 arena_prof_promoted(tsdn, p, usize);
2181 } else {
2182 p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
2183 slow_path);
2184 }
2185
2186 return (p);
2187 }
2188
2189 JEMALLOC_ALWAYS_INLINE_C void *
2190 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
2191 {
2192 void *p;
2193 size_t alignment;
2194 bool zero;
2195 tcache_t *tcache;
2196 arena_t *arena;
2197 prof_tctx_t *tctx;
2198
2199 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2200 &zero, &tcache, &arena)))
2201 return (NULL);
2202 tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);
2203 if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
2204 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero,
2205 tcache, arena, slow_path);
2206 } else if ((uintptr_t)tctx > (uintptr_t)1U) {
2207 p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero,
2208 tcache, arena, slow_path);
2209 } else
2210 p = NULL;
2211 if (unlikely(p == NULL)) {
2212 prof_alloc_rollback(tsd, tctx, true);
2213 return (NULL);
2214 }
2215 prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
2216
2217 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
2218 return (p);
2219 }
2220
2221 JEMALLOC_ALWAYS_INLINE_C void *
2222 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize,
2223 bool slow_path)
2224 {
2225 void *p;
2226 size_t alignment;
2227 bool zero;
2228 tcache_t *tcache;
2229 arena_t *arena;
2230
2231 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2232 &zero, &tcache, &arena)))
2233 return (NULL);
2234 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache,
2235 arena, slow_path);
2236 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
2237 return (p);
2238 }
2239
2240 /* This function guarantees that *tsdn is non-NULL on success. */
2241 JEMALLOC_ALWAYS_INLINE_C void *
2242 imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
2243 bool slow_path)
2244 {
2245 tsd_t *tsd;
2246
2247 if (slow_path && unlikely(malloc_init())) {
2248 *tsdn = NULL;
2249 return (NULL);
2250 }
2251
2252 tsd = tsd_fetch();
2253 *tsdn = tsd_tsdn(tsd);
2254 witness_assert_lockless(tsd_tsdn(tsd));
2255
2256 if (likely(flags == 0)) {
2257 szind_t ind = size2index(size);
2258 if (unlikely(ind >= NSIZES))
2259 return (NULL);
2260 if (config_stats || (config_prof && opt_prof) || (slow_path &&
2261 config_valgrind && unlikely(in_valgrind))) {
2262 *usize = index2size(ind);
2263 assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
2264 }
2265
2266 if (config_prof && opt_prof) {
2267 return (ialloc_prof(tsd, *usize, ind, false,
2268 slow_path));
2269 }
2270
2271 return (ialloc(tsd, size, ind, false, slow_path));
2272 }
2273
2274 if (config_prof && opt_prof)
2275 return (imallocx_prof(tsd, size, flags, usize, slow_path));
2276
2277 return (imallocx_no_prof(tsd, size, flags, usize, slow_path));
2278 }
2279
2280 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2281 void JEMALLOC_NOTHROW *
2282 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
2283 je_mallocx(size_t size, int flags)
2284 {
2285 tsdn_t *tsdn;
2286 void *p;
2287 size_t usize;
2288
2289 assert(size != 0);
2290
2291 if (likely(!malloc_slow)) {
2292 p = imallocx_body(size, flags, &tsdn, &usize, false);
2293 ialloc_post_check(p, tsdn, usize, "mallocx", false, false);
2294 } else {
2295 p = imallocx_body(size, flags, &tsdn, &usize, true);
2296 ialloc_post_check(p, tsdn, usize, "mallocx", false, true);
2297 UTRACE(0, size, p);
2298 JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize,
2299 MALLOCX_ZERO_GET(flags));
2300 }
2301
2302 return (p);
2303 }
2304
2305 static void *
2306 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize,
2307 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
2308 prof_tctx_t *tctx)
2309 {
2310 void *p;
2311
2312 if (tctx == NULL)
2313 return (NULL);
2314 if (usize <= SMALL_MAXCLASS) {
2315 p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment,
2316 zero, tcache, arena);
2317 if (p == NULL)
2318 return (NULL);
2319 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
2320 } else {
2321 p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
2322 tcache, arena);
2323 }
2324
2325 return (p);
2326 }
2327
2328 JEMALLOC_ALWAYS_INLINE_C void *
2329 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
2330 size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
2331 arena_t *arena)
2332 {
2333 void *p;
2334 bool prof_active;
2335 prof_tctx_t *old_tctx, *tctx;
2336
2337 prof_active = prof_active_get_unlocked();
2338 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
2339 tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
2340 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2341 p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
2342 alignment, zero, tcache, arena, tctx);
2343 } else {
2344 p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,
2345 tcache, arena);
2346 }
2347 if (unlikely(p == NULL)) {
2348 prof_alloc_rollback(tsd, tctx, false);
2349 return (NULL);
2350 }
2351
2352 if (p == old_ptr && alignment != 0) {
2353 /*
2354 * The allocation did not move, so it is possible that the size
2355 * class is smaller than would guarantee the requested
2356 * alignment, and that the alignment constraint was
2357 * serendipitously satisfied. Additionally, old_usize may not
2358 * be the same as the current usize because of in-place large
2359 * reallocation. Therefore, query the actual value of usize.
2360 */
2361 *usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2362 }
2363 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
2364 old_usize, old_tctx);
2365
2366 return (p);
2367 }
2368
2369 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2370 void JEMALLOC_NOTHROW *
2371 JEMALLOC_ALLOC_SIZE(2)
2372 je_rallocx(void *ptr, size_t size, int flags)
2373 {
2374 void *p;
2375 tsd_t *tsd;
2376 size_t usize;
2377 size_t old_usize;
2378 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
2379 size_t alignment = MALLOCX_ALIGN_GET(flags);
2380 bool zero = flags & MALLOCX_ZERO;
2381 arena_t *arena;
2382 tcache_t *tcache;
2383
2384 assert(ptr != NULL);
2385 assert(size != 0);
2386 assert(malloc_initialized() || IS_INITIALIZER);
2387 malloc_thread_init();
2388 tsd = tsd_fetch();
2389 witness_assert_lockless(tsd_tsdn(tsd));
2390
2391 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
2392 unsigned arena_ind = MALLOCX_ARENA_GET(flags);
2393 arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
2394 if (unlikely(arena == NULL))
2395 goto label_oom;
2396 } else
2397 arena = NULL;
2398
2399 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
2400 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2401 tcache = NULL;
2402 else
2403 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2404 } else
2405 tcache = tcache_get(tsd, true);
2406
2407 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
2408 if (config_valgrind && unlikely(in_valgrind))
2409 old_rzsize = u2rz(old_usize);
2410
2411 if (config_prof && opt_prof) {
2412 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
2413 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
2414 goto label_oom;
2415 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
2416 zero, tcache, arena);
2417 if (unlikely(p == NULL))
2418 goto label_oom;
2419 } else {
2420 p = iralloct(tsd, ptr, old_usize, size, alignment, zero,
2421 tcache, arena);
2422 if (unlikely(p == NULL))
2423 goto label_oom;
2424 if (config_stats || (config_valgrind && unlikely(in_valgrind)))
2425 usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2426 }
2427 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
2428
2429 if (config_stats) {
2430 *tsd_thread_allocatedp_get(tsd) += usize;
2431 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2432 }
2433 UTRACE(ptr, size, p);
2434 JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr,
2435 old_usize, old_rzsize, no, zero);
2436 witness_assert_lockless(tsd_tsdn(tsd));
2437 return (p);
2438 label_oom:
2439 if (config_xmalloc && unlikely(opt_xmalloc)) {
2440 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
2441 abort();
2442 }
2443 UTRACE(ptr, size, 0);
2444 witness_assert_lockless(tsd_tsdn(tsd));
2445 return (NULL);
2446 }
2447
2448 JEMALLOC_ALWAYS_INLINE_C size_t
2449 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
2450 size_t extra, size_t alignment, bool zero)
2451 {
2452 size_t usize;
2453
2454 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero))
2455 return (old_usize);
2456 usize = isalloc(tsdn, ptr, config_prof);
2457
2458 return (usize);
2459 }
2460
2461 static size_t
2462 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
2463 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx)
2464 {
2465 size_t usize;
2466
2467 if (tctx == NULL)
2468 return (old_usize);
2469 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
2470 zero);
2471
2472 return (usize);
2473 }
2474
2475 JEMALLOC_ALWAYS_INLINE_C size_t
2476 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
2477 size_t extra, size_t alignment, bool zero)
2478 {
2479 size_t usize_max, usize;
2480 bool prof_active;
2481 prof_tctx_t *old_tctx, *tctx;
2482
2483 prof_active = prof_active_get_unlocked();
2484 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
2485 /*
2486 * usize isn't knowable before ixalloc() returns when extra is non-zero.
2487 * Therefore, compute its maximum possible value and use that in
2488 * prof_alloc_prep() to decide whether to capture a backtrace.
2489 * prof_realloc() will use the actual usize to decide whether to sample.
2490 */
2491 if (alignment == 0) {
2492 usize_max = s2u(size+extra);
2493 assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS);
2494 } else {
2495 usize_max = sa2u(size+extra, alignment);
2496 if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) {
2497 /*
2498 * usize_max is out of range, and chances are that
2499 * allocation will fail, but use the maximum possible
2500 * value and carry on with prof_alloc_prep(), just in
2501 * case allocation succeeds.
2502 */
2503 usize_max = HUGE_MAXCLASS;
2504 }
2505 }
2506 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
2507
2508 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2509 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
2510 size, extra, alignment, zero, tctx);
2511 } else {
2512 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2513 extra, alignment, zero);
2514 }
2515 if (usize == old_usize) {
2516 prof_alloc_rollback(tsd, tctx, false);
2517 return (usize);
2518 }
2519 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
2520 old_tctx);
2521
2522 return (usize);
2523 }
2524
2525 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2526 je_xallocx(void *ptr, size_t size, size_t extra, int flags)
2527 {
2528 tsd_t *tsd;
2529 size_t usize, old_usize;
2530 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
2531 size_t alignment = MALLOCX_ALIGN_GET(flags);
2532 bool zero = flags & MALLOCX_ZERO;
2533
2534 assert(ptr != NULL);
2535 assert(size != 0);
2536 assert(SIZE_T_MAX - size >= extra);
2537 assert(malloc_initialized() || IS_INITIALIZER);
2538 malloc_thread_init();
2539 tsd = tsd_fetch();
2540 witness_assert_lockless(tsd_tsdn(tsd));
2541
2542 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
2543
2544 /*
2545 * The API explicitly absolves itself of protecting against (size +
2546 * extra) numerical overflow, but we may need to clamp extra to avoid
2547 * exceeding HUGE_MAXCLASS.
2548 *
2549 * Ordinarily, size limit checking is handled deeper down, but here we
2550 * have to check as part of (size + extra) clamping, since we need the
2551 * clamped value in the above helper functions.
2552 */
2553 if (unlikely(size > HUGE_MAXCLASS)) {
2554 usize = old_usize;
2555 goto label_not_resized;
2556 }
2557 if (unlikely(HUGE_MAXCLASS - size < extra))
2558 extra = HUGE_MAXCLASS - size;
2559
2560 if (config_valgrind && unlikely(in_valgrind))
2561 old_rzsize = u2rz(old_usize);
2562
2563 if (config_prof && opt_prof) {
2564 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
2565 alignment, zero);
2566 } else {
2567 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2568 extra, alignment, zero);
2569 }
2570 if (unlikely(usize == old_usize))
2571 goto label_not_resized;
2572
2573 if (config_stats) {
2574 *tsd_thread_allocatedp_get(tsd) += usize;
2575 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2576 }
2577 JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr,
2578 old_usize, old_rzsize, no, zero);
2579 label_not_resized:
2580 UTRACE(ptr, size, ptr);
2581 witness_assert_lockless(tsd_tsdn(tsd));
2582 return (usize);
2583 }
2584
2585 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2586 JEMALLOC_ATTR(pure)
2587 je_sallocx(const void *ptr, int flags)
2588 {
2589 size_t usize;
2590 tsdn_t *tsdn;
2591
2592 assert(malloc_initialized() || IS_INITIALIZER);
2593 malloc_thread_init();
2594
2595 tsdn = tsdn_fetch();
2596 witness_assert_lockless(tsdn);
2597
2598 if (config_ivsalloc)
2599 usize = ivsalloc(tsdn, ptr, config_prof);
2600 else
2601 usize = isalloc(tsdn, ptr, config_prof);
2602
2603 witness_assert_lockless(tsdn);
2604 return (usize);
2605 }
2606
2607 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2608 je_dallocx(void *ptr, int flags)
2609 {
2610 tsd_t *tsd;
2611 tcache_t *tcache;
2612
2613 assert(ptr != NULL);
2614 assert(malloc_initialized() || IS_INITIALIZER);
2615
2616 tsd = tsd_fetch();
2617 witness_assert_lockless(tsd_tsdn(tsd));
2618 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
2619 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2620 tcache = NULL;
2621 else
2622 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2623 } else
2624 tcache = tcache_get(tsd, false);
2625
2626 UTRACE(ptr, 0, 0);
2627 if (likely(!malloc_slow))
2628 ifree(tsd, ptr, tcache, false);
2629 else
2630 ifree(tsd, ptr, tcache, true);
2631 witness_assert_lockless(tsd_tsdn(tsd));
2632 }
2633
2634 JEMALLOC_ALWAYS_INLINE_C size_t
2635 inallocx(tsdn_t *tsdn, size_t size, int flags)
2636 {
2637 size_t usize;
2638
2639 witness_assert_lockless(tsdn);
2640
2641 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0))
2642 usize = s2u(size);
2643 else
2644 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
2645 witness_assert_lockless(tsdn);
2646 return (usize);
2647 }
2648
2649 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2650 je_sdallocx(void *ptr, size_t size, int flags)
2651 {
2652 tsd_t *tsd;
2653 tcache_t *tcache;
2654 size_t usize;
2655
2656 assert(ptr != NULL);
2657 assert(malloc_initialized() || IS_INITIALIZER);
2658 tsd = tsd_fetch();
2659 usize = inallocx(tsd_tsdn(tsd), size, flags);
2660 assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
2661
2662 witness_assert_lockless(tsd_tsdn(tsd));
2663 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
2664 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
2665 tcache = NULL;
2666 else
2667 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2668 } else
2669 tcache = tcache_get(tsd, false);
2670
2671 UTRACE(ptr, 0, 0);
2672 if (likely(!malloc_slow))
2673 isfree(tsd, ptr, usize, tcache, false);
2674 else
2675 isfree(tsd, ptr, usize, tcache, true);
2676 witness_assert_lockless(tsd_tsdn(tsd));
2677 }
2678
2679 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2680 JEMALLOC_ATTR(pure)
2681 je_nallocx(size_t size, int flags)
2682 {
2683 size_t usize;
2684 tsdn_t *tsdn;
2685
2686 assert(size != 0);
2687
2688 if (unlikely(malloc_init()))
2689 return (0);
2690
2691 tsdn = tsdn_fetch();
2692 witness_assert_lockless(tsdn);
2693
2694 usize = inallocx(tsdn, size, flags);
2695 if (unlikely(usize > HUGE_MAXCLASS))
2696 return (0);
2697
2698 witness_assert_lockless(tsdn);
2699 return (usize);
2700 }
2701
2702 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2703 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
2704 size_t newlen)
2705 {
2706 int ret;
2707 tsd_t *tsd;
2708
2709 if (unlikely(malloc_init()))
2710 return (EAGAIN);
2711
2712 tsd = tsd_fetch();
2713 witness_assert_lockless(tsd_tsdn(tsd));
2714 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
2715 witness_assert_lockless(tsd_tsdn(tsd));
2716 return (ret);
2717 }
2718
2719 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2720 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
2721 {
2722 int ret;
2723 tsdn_t *tsdn;
2724
2725 if (unlikely(malloc_init()))
2726 return (EAGAIN);
2727
2728 tsdn = tsdn_fetch();
2729 witness_assert_lockless(tsdn);
2730 ret = ctl_nametomib(tsdn, name, mibp, miblenp);
2731 witness_assert_lockless(tsdn);
2732 return (ret);
2733 }
2734
2735 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2736 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
2737 void *newp, size_t newlen)
2738 {
2739 int ret;
2740 tsd_t *tsd;
2741
2742 if (unlikely(malloc_init()))
2743 return (EAGAIN);
2744
2745 tsd = tsd_fetch();
2746 witness_assert_lockless(tsd_tsdn(tsd));
2747 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
2748 witness_assert_lockless(tsd_tsdn(tsd));
2749 return (ret);
2750 }
2751
2752 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2753 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
2754 const char *opts)
2755 {
2756 tsdn_t *tsdn;
2757
2758 tsdn = tsdn_fetch();
2759 witness_assert_lockless(tsdn);
2760 stats_print(write_cb, cbopaque, opts);
2761 witness_assert_lockless(tsdn);
2762 }
2763
2764 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
2765 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
2766 {
2767 size_t ret;
2768 tsdn_t *tsdn;
2769
2770 assert(malloc_initialized() || IS_INITIALIZER);
2771 malloc_thread_init();
2772
2773 tsdn = tsdn_fetch();
2774 witness_assert_lockless(tsdn);
2775
2776 if (config_ivsalloc)
2777 ret = ivsalloc(tsdn, ptr, config_prof);
2778 else
2779 ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
2780
2781 witness_assert_lockless(tsdn);
2782 return (ret);
2783 }
2784
2785 /*
2786 * End non-standard functions.
2787 */
2788 /******************************************************************************/
2789 /*
2790 * The following functions are used by threading libraries for protection of
2791 * malloc during fork().
2792 */
2793
2794 /*
2795 * If an application creates a thread before doing any allocation in the main
2796 * thread, then calls fork(2) in the main thread followed by memory allocation
2797 * in the child process, a race can occur that results in deadlock within the
2798 * child: the main thread may have forked while the created thread had
2799 * partially initialized the allocator. Ordinarily jemalloc prevents
2800 * fork/malloc races via the following functions it registers during
2801 * initialization using pthread_atfork(), but of course that does no good if
2802 * the allocator isn't fully initialized at fork time. The following library
2803 * constructor is a partial solution to this problem. It may still be possible
2804 * to trigger the deadlock described above, but doing so would involve forking
2805 * via a library constructor that runs before jemalloc's runs.
2806 */
2807 #ifndef JEMALLOC_JET
2808 JEMALLOC_ATTR(constructor)
2809 static void
2810 jemalloc_constructor(void)
2811 {
2812
2813 malloc_init();
2814 }
2815 #endif
2816
2817 #ifndef JEMALLOC_MUTEX_INIT_CB
2818 void
2819 jemalloc_prefork(void)
2820 #else
2821 JEMALLOC_EXPORT void
2822 _malloc_prefork(void)
2823 #endif
2824 {
2825 tsd_t *tsd;
2826 unsigned i, j, narenas;
2827 arena_t *arena;
2828
2829 #ifdef JEMALLOC_MUTEX_INIT_CB
2830 if (!malloc_initialized())
2831 return;
2832 #endif
2833 assert(malloc_initialized());
2834
2835 tsd = tsd_fetch();
2836
2837 narenas = narenas_total_get();
2838
2839 witness_prefork(tsd);
2840 /* Acquire all mutexes in a safe order. */
2841 ctl_prefork(tsd_tsdn(tsd));
2842 tcache_prefork(tsd_tsdn(tsd));
2843 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
2844 prof_prefork0(tsd_tsdn(tsd));
2845 for (i = 0; i < 3; i++) {
2846 for (j = 0; j < narenas; j++) {
2847 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
2848 NULL) {
2849 switch (i) {
2850 case 0:
2851 arena_prefork0(tsd_tsdn(tsd), arena);
2852 break;
2853 case 1:
2854 arena_prefork1(tsd_tsdn(tsd), arena);
2855 break;
2856 case 2:
2857 arena_prefork2(tsd_tsdn(tsd), arena);
2858 break;
2859 default: not_reached();
2860 }
2861 }
2862 }
2863 }
2864 base_prefork(tsd_tsdn(tsd));
2865 for (i = 0; i < narenas; i++) {
2866 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2867 arena_prefork3(tsd_tsdn(tsd), arena);
2868 }
2869 prof_prefork1(tsd_tsdn(tsd));
2870 }
2871
2872 #ifndef JEMALLOC_MUTEX_INIT_CB
2873 void
2874 jemalloc_postfork_parent(void)
2875 #else
2876 JEMALLOC_EXPORT void
2877 _malloc_postfork(void)
2878 #endif
2879 {
2880 tsd_t *tsd;
2881 unsigned i, narenas;
2882
2883 #ifdef JEMALLOC_MUTEX_INIT_CB
2884 if (!malloc_initialized())
2885 return;
2886 #endif
2887 assert(malloc_initialized());
2888
2889 tsd = tsd_fetch();
2890
2891 witness_postfork_parent(tsd);
2892 /* Release all mutexes, now that fork() has completed. */
2893 base_postfork_parent(tsd_tsdn(tsd));
2894 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
2895 arena_t *arena;
2896
2897 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2898 arena_postfork_parent(tsd_tsdn(tsd), arena);
2899 }
2900 prof_postfork_parent(tsd_tsdn(tsd));
2901 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
2902 tcache_postfork_parent(tsd_tsdn(tsd));
2903 ctl_postfork_parent(tsd_tsdn(tsd));
2904 }
2905
2906 void
2907 jemalloc_postfork_child(void)
2908 {
2909 tsd_t *tsd;
2910 unsigned i, narenas;
2911
2912 assert(malloc_initialized());
2913
2914 tsd = tsd_fetch();
2915
2916 witness_postfork_child(tsd);
2917 /* Release all mutexes, now that fork() has completed. */
2918 base_postfork_child(tsd_tsdn(tsd));
2919 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
2920 arena_t *arena;
2921
2922 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2923 arena_postfork_child(tsd_tsdn(tsd), arena);
2924 }
2925 prof_postfork_child(tsd_tsdn(tsd));
2926 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
2927 tcache_postfork_child(tsd_tsdn(tsd));
2928 ctl_postfork_child(tsd_tsdn(tsd));
2929 }
2930
2931 /******************************************************************************/
2932
2933 /* ANDROID extension */
2934 #include "android_je_iterate.c"
2935 #include "android_je_mallinfo.c"
2936 /* End ANDROID extension */
2937