1 #define JEMALLOC_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /******************************************************************************/
5 /* Data. */
6
7 malloc_tsd_data(, arenas, arena_t *, NULL)
8 malloc_tsd_data(, thread_allocated, thread_allocated_t,
9 THREAD_ALLOCATED_INITIALIZER)
10
11 /* Runtime configuration options. */
12 const char *je_malloc_conf;
13 bool opt_abort =
14 #ifdef JEMALLOC_DEBUG
15 true
16 #else
17 false
18 #endif
19 ;
20 bool opt_junk =
21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
22 true
23 #else
24 false
25 #endif
26 ;
27 size_t opt_quarantine = ZU(0);
28 bool opt_redzone = false;
29 bool opt_utrace = false;
30 bool opt_xmalloc = false;
31 bool opt_zero = false;
32 size_t opt_narenas = 0;
33
34 /* Initialized to true if the process is running inside Valgrind. */
35 bool in_valgrind;
36
37 unsigned ncpus;
38
39 malloc_mutex_t arenas_lock;
40 arena_t **arenas;
41 unsigned narenas_total;
42 unsigned narenas_auto;
43
44 /* Set to true once the allocator has been initialized. */
45 static bool malloc_initialized = false;
46
47 #ifdef JEMALLOC_THREADED_INIT
48 /* Used to let the initializing thread recursively allocate. */
49 # define NO_INITIALIZER ((unsigned long)0)
50 # define INITIALIZER pthread_self()
51 # define IS_INITIALIZER (malloc_initializer == pthread_self())
52 static pthread_t malloc_initializer = NO_INITIALIZER;
53 #else
54 # define NO_INITIALIZER false
55 # define INITIALIZER true
56 # define IS_INITIALIZER malloc_initializer
57 static bool malloc_initializer = NO_INITIALIZER;
58 #endif
59
60 /* Used to avoid initialization races. */
61 #ifdef _WIN32
62 static malloc_mutex_t init_lock;
63
JEMALLOC_ATTR(constructor)64 JEMALLOC_ATTR(constructor)
65 static void WINAPI
66 _init_init_lock(void)
67 {
68
69 malloc_mutex_init(&init_lock);
70 }
71
72 #ifdef _MSC_VER
73 # pragma section(".CRT$XCU", read)
74 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
75 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
76 #endif
77
78 #else
79 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
80 #endif
81
82 typedef struct {
83 void *p; /* Input pointer (as in realloc(p, s)). */
84 size_t s; /* Request size. */
85 void *r; /* Result pointer. */
86 } malloc_utrace_t;
87
88 #ifdef JEMALLOC_UTRACE
89 # define UTRACE(a, b, c) do { \
90 if (opt_utrace) { \
91 int utrace_serrno = errno; \
92 malloc_utrace_t ut; \
93 ut.p = (a); \
94 ut.s = (b); \
95 ut.r = (c); \
96 utrace(&ut, sizeof(ut)); \
97 errno = utrace_serrno; \
98 } \
99 } while (0)
100 #else
101 # define UTRACE(a, b, c)
102 #endif
103
104 /******************************************************************************/
105 /*
106 * Function prototypes for static functions that are referenced prior to
107 * definition.
108 */
109
110 static bool malloc_init_hard(void);
111
112 /******************************************************************************/
113 /*
114 * Begin miscellaneous support functions.
115 */
116
117 /* Create a new arena and insert it into the arenas array at index ind. */
118 arena_t *
arenas_extend(unsigned ind)119 arenas_extend(unsigned ind)
120 {
121 arena_t *ret;
122
123 ret = (arena_t *)base_alloc(sizeof(arena_t));
124 if (ret != NULL && arena_new(ret, ind) == false) {
125 arenas[ind] = ret;
126 return (ret);
127 }
128 /* Only reached if there is an OOM error. */
129
130 /*
131 * OOM here is quite inconvenient to propagate, since dealing with it
132 * would require a check for failure in the fast path. Instead, punt
133 * by using arenas[0]. In practice, this is an extremely unlikely
134 * failure.
135 */
136 malloc_write("<jemalloc>: Error initializing arena\n");
137 if (opt_abort)
138 abort();
139
140 return (arenas[0]);
141 }
142
143 /* Slow path, called only by choose_arena(). */
144 arena_t *
choose_arena_hard(void)145 choose_arena_hard(void)
146 {
147 arena_t *ret;
148
149 if (narenas_auto > 1) {
150 unsigned i, choose, first_null;
151
152 choose = 0;
153 first_null = narenas_auto;
154 malloc_mutex_lock(&arenas_lock);
155 assert(arenas[0] != NULL);
156 for (i = 1; i < narenas_auto; i++) {
157 if (arenas[i] != NULL) {
158 /*
159 * Choose the first arena that has the lowest
160 * number of threads assigned to it.
161 */
162 if (arenas[i]->nthreads <
163 arenas[choose]->nthreads)
164 choose = i;
165 } else if (first_null == narenas_auto) {
166 /*
167 * Record the index of the first uninitialized
168 * arena, in case all extant arenas are in use.
169 *
170 * NB: It is possible for there to be
171 * discontinuities in terms of initialized
172 * versus uninitialized arenas, due to the
173 * "thread.arena" mallctl.
174 */
175 first_null = i;
176 }
177 }
178
179 if (arenas[choose]->nthreads == 0
180 || first_null == narenas_auto) {
181 /*
182 * Use an unloaded arena, or the least loaded arena if
183 * all arenas are already initialized.
184 */
185 ret = arenas[choose];
186 } else {
187 /* Initialize a new arena. */
188 ret = arenas_extend(first_null);
189 }
190 ret->nthreads++;
191 malloc_mutex_unlock(&arenas_lock);
192 } else {
193 ret = arenas[0];
194 malloc_mutex_lock(&arenas_lock);
195 ret->nthreads++;
196 malloc_mutex_unlock(&arenas_lock);
197 }
198
199 arenas_tsd_set(&ret);
200
201 return (ret);
202 }
203
204 static void
stats_print_atexit(void)205 stats_print_atexit(void)
206 {
207
208 if (config_tcache && config_stats) {
209 unsigned narenas, i;
210
211 /*
212 * Merge stats from extant threads. This is racy, since
213 * individual threads do not lock when recording tcache stats
214 * events. As a consequence, the final stats may be slightly
215 * out of date by the time they are reported, if other threads
216 * continue to allocate.
217 */
218 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
219 arena_t *arena = arenas[i];
220 if (arena != NULL) {
221 tcache_t *tcache;
222
223 /*
224 * tcache_stats_merge() locks bins, so if any
225 * code is introduced that acquires both arena
226 * and bin locks in the opposite order,
227 * deadlocks may result.
228 */
229 malloc_mutex_lock(&arena->lock);
230 ql_foreach(tcache, &arena->tcache_ql, link) {
231 tcache_stats_merge(tcache, arena);
232 }
233 malloc_mutex_unlock(&arena->lock);
234 }
235 }
236 }
237 je_malloc_stats_print(NULL, NULL, NULL);
238 }
239
240 /*
241 * End miscellaneous support functions.
242 */
243 /******************************************************************************/
244 /*
245 * Begin initialization functions.
246 */
247
248 static unsigned
malloc_ncpus(void)249 malloc_ncpus(void)
250 {
251 long result;
252
253 #ifdef _WIN32
254 SYSTEM_INFO si;
255 GetSystemInfo(&si);
256 result = si.dwNumberOfProcessors;
257 #else
258 result = sysconf(_SC_NPROCESSORS_ONLN);
259 #endif
260 return ((result == -1) ? 1 : (unsigned)result);
261 }
262
263 void
arenas_cleanup(void * arg)264 arenas_cleanup(void *arg)
265 {
266 arena_t *arena = *(arena_t **)arg;
267
268 malloc_mutex_lock(&arenas_lock);
269 arena->nthreads--;
270 malloc_mutex_unlock(&arenas_lock);
271 }
272
273 JEMALLOC_ALWAYS_INLINE_C void
malloc_thread_init(void)274 malloc_thread_init(void)
275 {
276
277 /*
278 * TSD initialization can't be safely done as a side effect of
279 * deallocation, because it is possible for a thread to do nothing but
280 * deallocate its TLS data via free(), in which case writing to TLS
281 * would cause write-after-free memory corruption. The quarantine
282 * facility *only* gets used as a side effect of deallocation, so make
283 * a best effort attempt at initializing its TSD by hooking all
284 * allocation events.
285 */
286 if (config_fill && opt_quarantine)
287 quarantine_alloc_hook();
288 }
289
290 JEMALLOC_ALWAYS_INLINE_C bool
malloc_init(void)291 malloc_init(void)
292 {
293
294 if (malloc_initialized == false && malloc_init_hard())
295 return (true);
296 malloc_thread_init();
297
298 return (false);
299 }
300
301 static bool
malloc_conf_next(char const ** opts_p,char const ** k_p,size_t * klen_p,char const ** v_p,size_t * vlen_p)302 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
303 char const **v_p, size_t *vlen_p)
304 {
305 bool accept;
306 const char *opts = *opts_p;
307
308 *k_p = opts;
309
310 for (accept = false; accept == false;) {
311 switch (*opts) {
312 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
313 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
314 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
315 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
316 case 'Y': case 'Z':
317 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
318 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
319 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
320 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
321 case 'y': case 'z':
322 case '0': case '1': case '2': case '3': case '4': case '5':
323 case '6': case '7': case '8': case '9':
324 case '_':
325 opts++;
326 break;
327 case ':':
328 opts++;
329 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
330 *v_p = opts;
331 accept = true;
332 break;
333 case '\0':
334 if (opts != *opts_p) {
335 malloc_write("<jemalloc>: Conf string ends "
336 "with key\n");
337 }
338 return (true);
339 default:
340 malloc_write("<jemalloc>: Malformed conf string\n");
341 return (true);
342 }
343 }
344
345 for (accept = false; accept == false;) {
346 switch (*opts) {
347 case ',':
348 opts++;
349 /*
350 * Look ahead one character here, because the next time
351 * this function is called, it will assume that end of
352 * input has been cleanly reached if no input remains,
353 * but we have optimistically already consumed the
354 * comma if one exists.
355 */
356 if (*opts == '\0') {
357 malloc_write("<jemalloc>: Conf string ends "
358 "with comma\n");
359 }
360 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
361 accept = true;
362 break;
363 case '\0':
364 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
365 accept = true;
366 break;
367 default:
368 opts++;
369 break;
370 }
371 }
372
373 *opts_p = opts;
374 return (false);
375 }
376
377 static void
malloc_conf_error(const char * msg,const char * k,size_t klen,const char * v,size_t vlen)378 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
379 size_t vlen)
380 {
381
382 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
383 (int)vlen, v);
384 }
385
386 static void
malloc_conf_init(void)387 malloc_conf_init(void)
388 {
389 unsigned i;
390 char buf[PATH_MAX + 1];
391 const char *opts, *k, *v;
392 size_t klen, vlen;
393
394 /*
395 * Automatically configure valgrind before processing options. The
396 * valgrind option remains in jemalloc 3.x for compatibility reasons.
397 */
398 if (config_valgrind) {
399 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
400 if (config_fill && in_valgrind) {
401 opt_junk = false;
402 assert(opt_zero == false);
403 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
404 opt_redzone = true;
405 }
406 if (config_tcache && in_valgrind)
407 opt_tcache = false;
408 }
409
410 #if defined(__ANDROID__)
411 /* Android only supports compiled options. */
412 for (i = 0; i < 1; i++) {
413 #else
414 for (i = 0; i < 3; i++) {
415 #endif
416 /* Get runtime configuration. */
417 switch (i) {
418 case 0:
419 if (je_malloc_conf != NULL) {
420 /*
421 * Use options that were compiled into the
422 * program.
423 */
424 opts = je_malloc_conf;
425 } else {
426 /* No configuration specified. */
427 buf[0] = '\0';
428 opts = buf;
429 }
430 break;
431 case 1: {
432 int linklen = 0;
433 #ifndef _WIN32
434 int saved_errno = errno;
435 const char *linkname =
436 # ifdef JEMALLOC_PREFIX
437 "/etc/"JEMALLOC_PREFIX"malloc.conf"
438 # else
439 "/etc/malloc.conf"
440 # endif
441 ;
442
443 /*
444 * Try to use the contents of the "/etc/malloc.conf"
445 * symbolic link's name.
446 */
447 linklen = readlink(linkname, buf, sizeof(buf) - 1);
448 if (linklen == -1) {
449 /* No configuration specified. */
450 linklen = 0;
451 /* restore errno */
452 set_errno(saved_errno);
453 }
454 #endif
455 buf[linklen] = '\0';
456 opts = buf;
457 break;
458 } case 2: {
459 const char *envname =
460 #ifdef JEMALLOC_PREFIX
461 JEMALLOC_CPREFIX"MALLOC_CONF"
462 #else
463 "MALLOC_CONF"
464 #endif
465 ;
466
467 if ((opts = getenv(envname)) != NULL) {
468 /*
469 * Do nothing; opts is already initialized to
470 * the value of the MALLOC_CONF environment
471 * variable.
472 */
473 } else {
474 /* No configuration specified. */
475 buf[0] = '\0';
476 opts = buf;
477 }
478 break;
479 } default:
480 not_reached();
481 buf[0] = '\0';
482 opts = buf;
483 }
484
485 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
486 &vlen) == false) {
487 #define CONF_MATCH(n) \
488 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
489 #define CONF_HANDLE_BOOL(o, n, cont) \
490 if (CONF_MATCH(n)) { \
491 if (strncmp("true", v, vlen) == 0 && \
492 vlen == sizeof("true")-1) \
493 o = true; \
494 else if (strncmp("false", v, vlen) == \
495 0 && vlen == sizeof("false")-1) \
496 o = false; \
497 else { \
498 malloc_conf_error( \
499 "Invalid conf value", \
500 k, klen, v, vlen); \
501 } \
502 if (cont) \
503 continue; \
504 }
505 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \
506 if (CONF_MATCH(n)) { \
507 uintmax_t um; \
508 char *end; \
509 \
510 set_errno(0); \
511 um = malloc_strtoumax(v, &end, 0); \
512 if (get_errno() != 0 || (uintptr_t)end -\
513 (uintptr_t)v != vlen) { \
514 malloc_conf_error( \
515 "Invalid conf value", \
516 k, klen, v, vlen); \
517 } else if (clip) { \
518 if (min != 0 && um < min) \
519 o = min; \
520 else if (um > max) \
521 o = max; \
522 else \
523 o = um; \
524 } else { \
525 if ((min != 0 && um < min) || \
526 um > max) { \
527 malloc_conf_error( \
528 "Out-of-range " \
529 "conf value", \
530 k, klen, v, vlen); \
531 } else \
532 o = um; \
533 } \
534 continue; \
535 }
536 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \
537 if (CONF_MATCH(n)) { \
538 long l; \
539 char *end; \
540 \
541 set_errno(0); \
542 l = strtol(v, &end, 0); \
543 if (get_errno() != 0 || (uintptr_t)end -\
544 (uintptr_t)v != vlen) { \
545 malloc_conf_error( \
546 "Invalid conf value", \
547 k, klen, v, vlen); \
548 } else if (l < (ssize_t)min || l > \
549 (ssize_t)max) { \
550 malloc_conf_error( \
551 "Out-of-range conf value", \
552 k, klen, v, vlen); \
553 } else \
554 o = l; \
555 continue; \
556 }
557 #define CONF_HANDLE_CHAR_P(o, n, d) \
558 if (CONF_MATCH(n)) { \
559 size_t cpylen = (vlen <= \
560 sizeof(o)-1) ? vlen : \
561 sizeof(o)-1; \
562 strncpy(o, v, cpylen); \
563 o[cpylen] = '\0'; \
564 continue; \
565 }
566
567 CONF_HANDLE_BOOL(opt_abort, "abort", true)
568 /*
569 * Chunks always require at least one header page, plus
570 * one data page in the absence of redzones, or three
571 * pages in the presence of redzones. In order to
572 * simplify options processing, fix the limit based on
573 * config_fill.
574 */
575 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
576 (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
577 true)
578 if (strncmp("dss", k, klen) == 0) {
579 int i;
580 bool match = false;
581 for (i = 0; i < dss_prec_limit; i++) {
582 if (strncmp(dss_prec_names[i], v, vlen)
583 == 0) {
584 if (chunk_dss_prec_set(i)) {
585 malloc_conf_error(
586 "Error setting dss",
587 k, klen, v, vlen);
588 } else {
589 opt_dss =
590 dss_prec_names[i];
591 match = true;
592 break;
593 }
594 }
595 }
596 if (match == false) {
597 malloc_conf_error("Invalid conf value",
598 k, klen, v, vlen);
599 }
600 continue;
601 }
602 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
603 SIZE_T_MAX, false)
604 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
605 -1, (sizeof(size_t) << 3) - 1)
606 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
607 if (config_fill) {
608 CONF_HANDLE_BOOL(opt_junk, "junk", true)
609 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
610 0, SIZE_T_MAX, false)
611 CONF_HANDLE_BOOL(opt_redzone, "redzone", true)
612 CONF_HANDLE_BOOL(opt_zero, "zero", true)
613 }
614 if (config_utrace) {
615 CONF_HANDLE_BOOL(opt_utrace, "utrace", true)
616 }
617 if (config_xmalloc) {
618 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true)
619 }
620 if (config_tcache) {
621 CONF_HANDLE_BOOL(opt_tcache, "tcache",
622 !config_valgrind || !in_valgrind)
623 if (CONF_MATCH("tcache")) {
624 assert(config_valgrind && in_valgrind);
625 if (opt_tcache) {
626 opt_tcache = false;
627 malloc_conf_error(
628 "tcache cannot be enabled "
629 "while running inside Valgrind",
630 k, klen, v, vlen);
631 }
632 continue;
633 }
634 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
635 "lg_tcache_max", -1,
636 (sizeof(size_t) << 3) - 1)
637 }
638 if (config_prof) {
639 CONF_HANDLE_BOOL(opt_prof, "prof", true)
640 CONF_HANDLE_CHAR_P(opt_prof_prefix,
641 "prof_prefix", "jeprof")
642 CONF_HANDLE_BOOL(opt_prof_active, "prof_active",
643 true)
644 CONF_HANDLE_SSIZE_T(opt_lg_prof_sample,
645 "lg_prof_sample", 0,
646 (sizeof(uint64_t) << 3) - 1)
647 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
648 true)
649 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
650 "lg_prof_interval", -1,
651 (sizeof(uint64_t) << 3) - 1)
652 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump",
653 true)
654 CONF_HANDLE_BOOL(opt_prof_final, "prof_final",
655 true)
656 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak",
657 true)
658 }
659 malloc_conf_error("Invalid conf pair", k, klen, v,
660 vlen);
661 #undef CONF_MATCH
662 #undef CONF_HANDLE_BOOL
663 #undef CONF_HANDLE_SIZE_T
664 #undef CONF_HANDLE_SSIZE_T
665 #undef CONF_HANDLE_CHAR_P
666 }
667 }
668 }
669
670 static bool
671 malloc_init_hard(void)
672 {
673 arena_t *init_arenas[1];
674
675 malloc_mutex_lock(&init_lock);
676 if (malloc_initialized || IS_INITIALIZER) {
677 /*
678 * Another thread initialized the allocator before this one
679 * acquired init_lock, or this thread is the initializing
680 * thread, and it is recursively allocating.
681 */
682 malloc_mutex_unlock(&init_lock);
683 return (false);
684 }
685 #ifdef JEMALLOC_THREADED_INIT
686 if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) {
687 /* Busy-wait until the initializing thread completes. */
688 do {
689 malloc_mutex_unlock(&init_lock);
690 CPU_SPINWAIT;
691 malloc_mutex_lock(&init_lock);
692 } while (malloc_initialized == false);
693 malloc_mutex_unlock(&init_lock);
694 return (false);
695 }
696 #endif
697 malloc_initializer = INITIALIZER;
698
699 malloc_tsd_boot();
700 if (config_prof)
701 prof_boot0();
702
703 malloc_conf_init();
704
705 if (opt_stats_print) {
706 /* Print statistics at exit. */
707 if (atexit(stats_print_atexit) != 0) {
708 malloc_write("<jemalloc>: Error in atexit()\n");
709 if (opt_abort)
710 abort();
711 }
712 }
713
714 if (base_boot()) {
715 malloc_mutex_unlock(&init_lock);
716 return (true);
717 }
718
719 if (chunk_boot()) {
720 malloc_mutex_unlock(&init_lock);
721 return (true);
722 }
723
724 if (ctl_boot()) {
725 malloc_mutex_unlock(&init_lock);
726 return (true);
727 }
728
729 if (config_prof)
730 prof_boot1();
731
732 arena_boot();
733
734 if (config_tcache && tcache_boot0()) {
735 malloc_mutex_unlock(&init_lock);
736 return (true);
737 }
738
739 if (huge_boot()) {
740 malloc_mutex_unlock(&init_lock);
741 return (true);
742 }
743
744 if (malloc_mutex_init(&arenas_lock)) {
745 malloc_mutex_unlock(&init_lock);
746 return (true);
747 }
748
749 /*
750 * Create enough scaffolding to allow recursive allocation in
751 * malloc_ncpus().
752 */
753 narenas_total = narenas_auto = 1;
754 arenas = init_arenas;
755 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
756
757 /*
758 * Initialize one arena here. The rest are lazily created in
759 * choose_arena_hard().
760 */
761 arenas_extend(0);
762 if (arenas[0] == NULL) {
763 malloc_mutex_unlock(&init_lock);
764 return (true);
765 }
766
767 /* Initialize allocation counters before any allocations can occur. */
768 if (config_stats && thread_allocated_tsd_boot()) {
769 malloc_mutex_unlock(&init_lock);
770 return (true);
771 }
772
773 if (arenas_tsd_boot()) {
774 malloc_mutex_unlock(&init_lock);
775 return (true);
776 }
777
778 if (config_tcache && tcache_boot1()) {
779 malloc_mutex_unlock(&init_lock);
780 return (true);
781 }
782
783 if (config_fill && quarantine_boot()) {
784 malloc_mutex_unlock(&init_lock);
785 return (true);
786 }
787
788 if (config_prof && prof_boot2()) {
789 malloc_mutex_unlock(&init_lock);
790 return (true);
791 }
792
793 malloc_mutex_unlock(&init_lock);
794 /**********************************************************************/
795 /* Recursive allocation may follow. */
796
797 ncpus = malloc_ncpus();
798
799 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
800 && !defined(_WIN32) && !defined(__native_client__))
801 /* LinuxThreads's pthread_atfork() allocates. */
802 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
803 jemalloc_postfork_child) != 0) {
804 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
805 if (opt_abort)
806 abort();
807 }
808 #endif
809
810 /* Done recursively allocating. */
811 /**********************************************************************/
812 malloc_mutex_lock(&init_lock);
813
814 if (mutex_boot()) {
815 malloc_mutex_unlock(&init_lock);
816 return (true);
817 }
818
819 if (opt_narenas == 0) {
820 /*
821 * For SMP systems, create more than one arena per CPU by
822 * default.
823 */
824 if (ncpus > 1)
825 opt_narenas = ncpus << 2;
826 else
827 opt_narenas = 1;
828 }
829 #if defined(ANDROID_MAX_ARENAS)
830 /* Never create more than MAX_ARENAS arenas regardless of num_cpus.
831 * Extra arenas use more PSS and are not very useful unless
832 * lots of threads are allocing/freeing at the same time.
833 */
834 if (opt_narenas > ANDROID_MAX_ARENAS)
835 opt_narenas = ANDROID_MAX_ARENAS;
836 #endif
837 narenas_auto = opt_narenas;
838 /*
839 * Make sure that the arenas array can be allocated. In practice, this
840 * limit is enough to allow the allocator to function, but the ctl
841 * machinery will fail to allocate memory at far lower limits.
842 */
843 if (narenas_auto > chunksize / sizeof(arena_t *)) {
844 narenas_auto = chunksize / sizeof(arena_t *);
845 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
846 narenas_auto);
847 }
848 narenas_total = narenas_auto;
849
850 /* Allocate and initialize arenas. */
851 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total);
852 if (arenas == NULL) {
853 malloc_mutex_unlock(&init_lock);
854 return (true);
855 }
856 /*
857 * Zero the array. In practice, this should always be pre-zeroed,
858 * since it was just mmap()ed, but let's be sure.
859 */
860 memset(arenas, 0, sizeof(arena_t *) * narenas_total);
861 /* Copy the pointer to the one arena that was already initialized. */
862 arenas[0] = init_arenas[0];
863
864 malloc_initialized = true;
865 malloc_mutex_unlock(&init_lock);
866
867 return (false);
868 }
869
870 /*
871 * End initialization functions.
872 */
873 /******************************************************************************/
874 /*
875 * Begin malloc(3)-compatible functions.
876 */
877
878 static void *
879 imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
880 {
881 void *p;
882
883 if (cnt == NULL)
884 return (NULL);
885 if (usize <= SMALL_MAXCLASS) {
886 p = imalloc(SMALL_MAXCLASS+1);
887 if (p == NULL)
888 return (NULL);
889 arena_prof_promoted(p, usize);
890 } else
891 p = imalloc(usize);
892
893 return (p);
894 }
895
896 JEMALLOC_ALWAYS_INLINE_C void *
897 imalloc_prof(size_t usize)
898 {
899 void *p;
900 prof_thr_cnt_t *cnt;
901
902 PROF_ALLOC_PREP(usize, cnt);
903 if ((uintptr_t)cnt != (uintptr_t)1U)
904 p = imalloc_prof_sample(usize, cnt);
905 else
906 p = imalloc(usize);
907 if (p == NULL)
908 return (NULL);
909 prof_malloc(p, usize, cnt);
910
911 return (p);
912 }
913
914 JEMALLOC_ALWAYS_INLINE_C void *
915 imalloc_body(size_t size, size_t *usize)
916 {
917
918 if (malloc_init())
919 return (NULL);
920
921 if (config_prof && opt_prof) {
922 *usize = s2u(size);
923 return (imalloc_prof(*usize));
924 }
925
926 if (config_stats || (config_valgrind && in_valgrind))
927 *usize = s2u(size);
928 return (imalloc(size));
929 }
930
931 void *
932 je_malloc(size_t size)
933 {
934 void *ret;
935 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
936
937 if (size == 0)
938 size = 1;
939
940 ret = imalloc_body(size, &usize);
941 if (ret == NULL) {
942 if (config_xmalloc && opt_xmalloc) {
943 malloc_write("<jemalloc>: Error in malloc(): "
944 "out of memory\n");
945 abort();
946 }
947 set_errno(ENOMEM);
948 }
949 if (config_stats && ret != NULL) {
950 assert(usize == isalloc(ret, config_prof));
951 thread_allocated_tsd_get()->allocated += usize;
952 }
953 UTRACE(0, size, ret);
954 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
955 return (ret);
956 }
957
958 static void *
959 imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
960 {
961 void *p;
962
963 if (cnt == NULL)
964 return (NULL);
965 if (usize <= SMALL_MAXCLASS) {
966 assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0);
967 p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment,
968 false);
969 if (p == NULL)
970 return (NULL);
971 arena_prof_promoted(p, usize);
972 } else
973 p = ipalloc(usize, alignment, false);
974
975 return (p);
976 }
977
978 JEMALLOC_ALWAYS_INLINE_C void *
979 imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
980 {
981 void *p;
982
983 if ((uintptr_t)cnt != (uintptr_t)1U)
984 p = imemalign_prof_sample(alignment, usize, cnt);
985 else
986 p = ipalloc(usize, alignment, false);
987 if (p == NULL)
988 return (NULL);
989 prof_malloc(p, usize, cnt);
990
991 return (p);
992 }
993
994 JEMALLOC_ATTR(nonnull(1))
995 static int
996 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
997 {
998 int ret;
999 size_t usize;
1000 void *result;
1001
1002 assert(min_alignment != 0);
1003
1004 if (malloc_init()) {
1005 result = NULL;
1006 goto label_oom;
1007 } else {
1008 if (size == 0)
1009 size = 1;
1010
1011 /* Make sure that alignment is a large enough power of 2. */
1012 if (((alignment - 1) & alignment) != 0
1013 || (alignment < min_alignment)) {
1014 if (config_xmalloc && opt_xmalloc) {
1015 malloc_write("<jemalloc>: Error allocating "
1016 "aligned memory: invalid alignment\n");
1017 abort();
1018 }
1019 result = NULL;
1020 ret = EINVAL;
1021 goto label_return;
1022 }
1023
1024 usize = sa2u(size, alignment);
1025 if (usize == 0) {
1026 result = NULL;
1027 goto label_oom;
1028 }
1029
1030 if (config_prof && opt_prof) {
1031 prof_thr_cnt_t *cnt;
1032
1033 PROF_ALLOC_PREP(usize, cnt);
1034 result = imemalign_prof(alignment, usize, cnt);
1035 } else
1036 result = ipalloc(usize, alignment, false);
1037 if (result == NULL)
1038 goto label_oom;
1039 }
1040
1041 *memptr = result;
1042 ret = 0;
1043 label_return:
1044 if (config_stats && result != NULL) {
1045 assert(usize == isalloc(result, config_prof));
1046 thread_allocated_tsd_get()->allocated += usize;
1047 }
1048 UTRACE(0, size, result);
1049 return (ret);
1050 label_oom:
1051 assert(result == NULL);
1052 if (config_xmalloc && opt_xmalloc) {
1053 malloc_write("<jemalloc>: Error allocating aligned memory: "
1054 "out of memory\n");
1055 abort();
1056 }
1057 ret = ENOMEM;
1058 goto label_return;
1059 }
1060
1061 int
1062 je_posix_memalign(void **memptr, size_t alignment, size_t size)
1063 {
1064 int ret = imemalign(memptr, alignment, size, sizeof(void *));
1065 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
1066 config_prof), false);
1067 return (ret);
1068 }
1069
1070 void *
1071 je_aligned_alloc(size_t alignment, size_t size)
1072 {
1073 void *ret;
1074 int err;
1075
1076 if ((err = imemalign(&ret, alignment, size, 1)) != 0) {
1077 ret = NULL;
1078 set_errno(err);
1079 }
1080 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
1081 false);
1082 return (ret);
1083 }
1084
1085 static void *
1086 icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
1087 {
1088 void *p;
1089
1090 if (cnt == NULL)
1091 return (NULL);
1092 if (usize <= SMALL_MAXCLASS) {
1093 p = icalloc(SMALL_MAXCLASS+1);
1094 if (p == NULL)
1095 return (NULL);
1096 arena_prof_promoted(p, usize);
1097 } else
1098 p = icalloc(usize);
1099
1100 return (p);
1101 }
1102
1103 JEMALLOC_ALWAYS_INLINE_C void *
1104 icalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
1105 {
1106 void *p;
1107
1108 if ((uintptr_t)cnt != (uintptr_t)1U)
1109 p = icalloc_prof_sample(usize, cnt);
1110 else
1111 p = icalloc(usize);
1112 if (p == NULL)
1113 return (NULL);
1114 prof_malloc(p, usize, cnt);
1115
1116 return (p);
1117 }
1118
1119 void *
1120 je_calloc(size_t num, size_t size)
1121 {
1122 void *ret;
1123 size_t num_size;
1124 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1125
1126 if (malloc_init()) {
1127 num_size = 0;
1128 ret = NULL;
1129 goto label_return;
1130 }
1131
1132 num_size = num * size;
1133 if (num_size == 0) {
1134 if (num == 0 || size == 0)
1135 num_size = 1;
1136 else {
1137 ret = NULL;
1138 goto label_return;
1139 }
1140 /*
1141 * Try to avoid division here. We know that it isn't possible to
1142 * overflow during multiplication if neither operand uses any of the
1143 * most significant half of the bits in a size_t.
1144 */
1145 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
1146 && (num_size / size != num)) {
1147 /* size_t overflow. */
1148 ret = NULL;
1149 goto label_return;
1150 }
1151
1152 if (config_prof && opt_prof) {
1153 prof_thr_cnt_t *cnt;
1154
1155 usize = s2u(num_size);
1156 PROF_ALLOC_PREP(usize, cnt);
1157 ret = icalloc_prof(usize, cnt);
1158 } else {
1159 if (config_stats || (config_valgrind && in_valgrind))
1160 usize = s2u(num_size);
1161 ret = icalloc(num_size);
1162 }
1163
1164 label_return:
1165 if (ret == NULL) {
1166 if (config_xmalloc && opt_xmalloc) {
1167 malloc_write("<jemalloc>: Error in calloc(): out of "
1168 "memory\n");
1169 abort();
1170 }
1171 set_errno(ENOMEM);
1172 }
1173 if (config_stats && ret != NULL) {
1174 assert(usize == isalloc(ret, config_prof));
1175 thread_allocated_tsd_get()->allocated += usize;
1176 }
1177 UTRACE(0, num_size, ret);
1178 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
1179 return (ret);
1180 }
1181
1182 static void *
1183 irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt)
1184 {
1185 void *p;
1186
1187 if (cnt == NULL)
1188 return (NULL);
1189 if (usize <= SMALL_MAXCLASS) {
1190 p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false);
1191 if (p == NULL)
1192 return (NULL);
1193 arena_prof_promoted(p, usize);
1194 } else
1195 p = iralloc(oldptr, usize, 0, 0, false);
1196
1197 return (p);
1198 }
1199
1200 JEMALLOC_ALWAYS_INLINE_C void *
1201 irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt)
1202 {
1203 void *p;
1204 prof_ctx_t *old_ctx;
1205
1206 old_ctx = prof_ctx_get(oldptr);
1207 if ((uintptr_t)cnt != (uintptr_t)1U)
1208 p = irealloc_prof_sample(oldptr, usize, cnt);
1209 else
1210 p = iralloc(oldptr, usize, 0, 0, false);
1211 if (p == NULL)
1212 return (NULL);
1213 prof_realloc(p, usize, cnt, old_usize, old_ctx);
1214
1215 return (p);
1216 }
1217
1218 JEMALLOC_INLINE_C void
1219 ifree(void *ptr)
1220 {
1221 size_t usize;
1222 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1223
1224 assert(ptr != NULL);
1225 assert(malloc_initialized || IS_INITIALIZER);
1226
1227 if (config_prof && opt_prof) {
1228 usize = isalloc(ptr, config_prof);
1229 prof_free(ptr, usize);
1230 } else if (config_stats || config_valgrind)
1231 usize = isalloc(ptr, config_prof);
1232 if (config_stats)
1233 thread_allocated_tsd_get()->deallocated += usize;
1234 if (config_valgrind && in_valgrind)
1235 rzsize = p2rz(ptr);
1236 iqalloc(ptr);
1237 JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1238 }
1239
1240 void *
1241 je_realloc(void *ptr, size_t size)
1242 {
1243 void *ret;
1244 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1245 size_t old_usize = 0;
1246 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1247
1248 if (size == 0) {
1249 if (ptr != NULL) {
1250 /* realloc(ptr, 0) is equivalent to free(ptr). */
1251 UTRACE(ptr, 0, 0);
1252 ifree(ptr);
1253 return (NULL);
1254 }
1255 size = 1;
1256 }
1257
1258 if (ptr != NULL) {
1259 assert(malloc_initialized || IS_INITIALIZER);
1260 malloc_thread_init();
1261
1262 if ((config_prof && opt_prof) || config_stats ||
1263 (config_valgrind && in_valgrind))
1264 old_usize = isalloc(ptr, config_prof);
1265 if (config_valgrind && in_valgrind)
1266 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
1267
1268 if (config_prof && opt_prof) {
1269 prof_thr_cnt_t *cnt;
1270
1271 usize = s2u(size);
1272 PROF_ALLOC_PREP(usize, cnt);
1273 ret = irealloc_prof(ptr, old_usize, usize, cnt);
1274 } else {
1275 if (config_stats || (config_valgrind && in_valgrind))
1276 usize = s2u(size);
1277 ret = iralloc(ptr, size, 0, 0, false);
1278 }
1279 } else {
1280 /* realloc(NULL, size) is equivalent to malloc(size). */
1281 ret = imalloc_body(size, &usize);
1282 }
1283
1284 if (ret == NULL) {
1285 if (config_xmalloc && opt_xmalloc) {
1286 malloc_write("<jemalloc>: Error in realloc(): "
1287 "out of memory\n");
1288 abort();
1289 }
1290 set_errno(ENOMEM);
1291 }
1292 if (config_stats && ret != NULL) {
1293 thread_allocated_t *ta;
1294 assert(usize == isalloc(ret, config_prof));
1295 ta = thread_allocated_tsd_get();
1296 ta->allocated += usize;
1297 ta->deallocated += old_usize;
1298 }
1299 UTRACE(ptr, size, ret);
1300 JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize,
1301 old_rzsize, true, false);
1302 return (ret);
1303 }
1304
1305 void
1306 je_free(void *ptr)
1307 {
1308
1309 UTRACE(ptr, 0, 0);
1310 if (ptr != NULL)
1311 ifree(ptr);
1312 }
1313
1314 /*
1315 * End malloc(3)-compatible functions.
1316 */
1317 /******************************************************************************/
1318 /*
1319 * Begin non-standard override functions.
1320 */
1321
1322 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
1323 void *
1324 je_memalign(size_t alignment, size_t size)
1325 {
1326 void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1327 imemalign(&ret, alignment, size, 1);
1328 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1329 return (ret);
1330 }
1331 #endif
1332
1333 #ifdef JEMALLOC_OVERRIDE_VALLOC
1334 void *
1335 je_valloc(size_t size)
1336 {
1337 void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1338 imemalign(&ret, PAGE, size, 1);
1339 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1340 return (ret);
1341 }
1342 #endif
1343
1344 /*
1345 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
1346 * #define je_malloc malloc
1347 */
1348 #define malloc_is_malloc 1
1349 #define is_malloc_(a) malloc_is_ ## a
1350 #define is_malloc(a) is_malloc_(a)
1351
1352 #if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__))
1353 /*
1354 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1355 * to inconsistently reference libc's malloc(3)-compatible functions
1356 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1357 *
1358 * These definitions interpose hooks in glibc. The functions are actually
1359 * passed an extra argument for the caller return address, which will be
1360 * ignored.
1361 */
1362 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
1363 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
1364 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
1365 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
1366 je_memalign;
1367 #endif
1368
1369 /*
1370 * End non-standard override functions.
1371 */
1372 /******************************************************************************/
1373 /*
1374 * Begin non-standard functions.
1375 */
1376
1377 JEMALLOC_ALWAYS_INLINE_C void *
1378 imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
1379 arena_t *arena)
1380 {
1381
1382 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize,
1383 alignment)));
1384
1385 if (alignment != 0)
1386 return (ipalloct(usize, alignment, zero, try_tcache, arena));
1387 else if (zero)
1388 return (icalloct(usize, try_tcache, arena));
1389 else
1390 return (imalloct(usize, try_tcache, arena));
1391 }
1392
1393 static void *
1394 imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache,
1395 arena_t *arena, prof_thr_cnt_t *cnt)
1396 {
1397 void *p;
1398
1399 if (cnt == NULL)
1400 return (NULL);
1401 if (usize <= SMALL_MAXCLASS) {
1402 size_t usize_promoted = (alignment == 0) ?
1403 s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment);
1404 assert(usize_promoted != 0);
1405 p = imallocx(usize_promoted, alignment, zero, try_tcache,
1406 arena);
1407 if (p == NULL)
1408 return (NULL);
1409 arena_prof_promoted(p, usize);
1410 } else
1411 p = imallocx(usize, alignment, zero, try_tcache, arena);
1412
1413 return (p);
1414 }
1415
1416 JEMALLOC_ALWAYS_INLINE_C void *
1417 imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache,
1418 arena_t *arena, prof_thr_cnt_t *cnt)
1419 {
1420 void *p;
1421
1422 if ((uintptr_t)cnt != (uintptr_t)1U) {
1423 p = imallocx_prof_sample(usize, alignment, zero, try_tcache,
1424 arena, cnt);
1425 } else
1426 p = imallocx(usize, alignment, zero, try_tcache, arena);
1427 if (p == NULL)
1428 return (NULL);
1429 prof_malloc(p, usize, cnt);
1430
1431 return (p);
1432 }
1433
1434 void *
1435 je_mallocx(size_t size, int flags)
1436 {
1437 void *p;
1438 size_t usize;
1439 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1440 & (SIZE_T_MAX-1));
1441 bool zero = flags & MALLOCX_ZERO;
1442 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1443 arena_t *arena;
1444 bool try_tcache;
1445
1446 assert(size != 0);
1447
1448 if (malloc_init())
1449 goto label_oom;
1450
1451 if (arena_ind != UINT_MAX) {
1452 arena = arenas[arena_ind];
1453 try_tcache = false;
1454 } else {
1455 arena = NULL;
1456 try_tcache = true;
1457 }
1458
1459 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1460 assert(usize != 0);
1461
1462 if (config_prof && opt_prof) {
1463 prof_thr_cnt_t *cnt;
1464
1465 PROF_ALLOC_PREP(usize, cnt);
1466 p = imallocx_prof(usize, alignment, zero, try_tcache, arena,
1467 cnt);
1468 } else
1469 p = imallocx(usize, alignment, zero, try_tcache, arena);
1470 if (p == NULL)
1471 goto label_oom;
1472
1473 if (config_stats) {
1474 assert(usize == isalloc(p, config_prof));
1475 thread_allocated_tsd_get()->allocated += usize;
1476 }
1477 UTRACE(0, size, p);
1478 JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
1479 return (p);
1480 label_oom:
1481 if (config_xmalloc && opt_xmalloc) {
1482 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n");
1483 abort();
1484 }
1485 UTRACE(0, size, 0);
1486 return (NULL);
1487 }
1488
1489 static void *
1490 irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
1491 bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
1492 prof_thr_cnt_t *cnt)
1493 {
1494 void *p;
1495
1496 if (cnt == NULL)
1497 return (NULL);
1498 if (usize <= SMALL_MAXCLASS) {
1499 p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1500 size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero,
1501 try_tcache_alloc, try_tcache_dalloc, arena);
1502 if (p == NULL)
1503 return (NULL);
1504 arena_prof_promoted(p, usize);
1505 } else {
1506 p = iralloct(oldptr, size, 0, alignment, zero,
1507 try_tcache_alloc, try_tcache_dalloc, arena);
1508 }
1509
1510 return (p);
1511 }
1512
1513 JEMALLOC_ALWAYS_INLINE_C void *
1514 irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
1515 size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
1516 arena_t *arena, prof_thr_cnt_t *cnt)
1517 {
1518 void *p;
1519 prof_ctx_t *old_ctx;
1520
1521 old_ctx = prof_ctx_get(oldptr);
1522 if ((uintptr_t)cnt != (uintptr_t)1U)
1523 p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
1524 try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1525 else {
1526 p = iralloct(oldptr, size, 0, alignment, zero,
1527 try_tcache_alloc, try_tcache_dalloc, arena);
1528 }
1529 if (p == NULL)
1530 return (NULL);
1531
1532 if (p == oldptr && alignment != 0) {
1533 /*
1534 * The allocation did not move, so it is possible that the size
1535 * class is smaller than would guarantee the requested
1536 * alignment, and that the alignment constraint was
1537 * serendipitously satisfied. Additionally, old_usize may not
1538 * be the same as the current usize because of in-place large
1539 * reallocation. Therefore, query the actual value of usize.
1540 */
1541 *usize = isalloc(p, config_prof);
1542 }
1543 prof_realloc(p, *usize, cnt, old_usize, old_ctx);
1544
1545 return (p);
1546 }
1547
1548 void *
1549 je_rallocx(void *ptr, size_t size, int flags)
1550 {
1551 void *p;
1552 size_t usize, old_usize;
1553 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1554 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1555 & (SIZE_T_MAX-1));
1556 bool zero = flags & MALLOCX_ZERO;
1557 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1558 bool try_tcache_alloc, try_tcache_dalloc;
1559 arena_t *arena;
1560
1561 assert(ptr != NULL);
1562 assert(size != 0);
1563 assert(malloc_initialized || IS_INITIALIZER);
1564 malloc_thread_init();
1565
1566 if (arena_ind != UINT_MAX) {
1567 arena_chunk_t *chunk;
1568 try_tcache_alloc = false;
1569 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1570 try_tcache_dalloc = (chunk == ptr || chunk->arena !=
1571 arenas[arena_ind]);
1572 arena = arenas[arena_ind];
1573 } else {
1574 try_tcache_alloc = true;
1575 try_tcache_dalloc = true;
1576 arena = NULL;
1577 }
1578
1579 if ((config_prof && opt_prof) || config_stats ||
1580 (config_valgrind && in_valgrind))
1581 old_usize = isalloc(ptr, config_prof);
1582 if (config_valgrind && in_valgrind)
1583 old_rzsize = u2rz(old_usize);
1584
1585 if (config_prof && opt_prof) {
1586 prof_thr_cnt_t *cnt;
1587
1588 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1589 assert(usize != 0);
1590 PROF_ALLOC_PREP(usize, cnt);
1591 p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
1592 try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1593 if (p == NULL)
1594 goto label_oom;
1595 } else {
1596 p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc,
1597 try_tcache_dalloc, arena);
1598 if (p == NULL)
1599 goto label_oom;
1600 if (config_stats || (config_valgrind && in_valgrind))
1601 usize = isalloc(p, config_prof);
1602 }
1603
1604 if (config_stats) {
1605 thread_allocated_t *ta;
1606 ta = thread_allocated_tsd_get();
1607 ta->allocated += usize;
1608 ta->deallocated += old_usize;
1609 }
1610 UTRACE(ptr, size, p);
1611 JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize,
1612 old_rzsize, false, zero);
1613 return (p);
1614 label_oom:
1615 if (config_xmalloc && opt_xmalloc) {
1616 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
1617 abort();
1618 }
1619 UTRACE(ptr, size, 0);
1620 return (NULL);
1621 }
1622
1623 JEMALLOC_ALWAYS_INLINE_C size_t
1624 ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
1625 size_t alignment, bool zero, arena_t *arena)
1626 {
1627 size_t usize;
1628
1629 if (ixalloc(ptr, size, extra, alignment, zero))
1630 return (old_usize);
1631 usize = isalloc(ptr, config_prof);
1632
1633 return (usize);
1634 }
1635
1636 static size_t
1637 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
1638 size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1639 prof_thr_cnt_t *cnt)
1640 {
1641 size_t usize;
1642
1643 if (cnt == NULL)
1644 return (old_usize);
1645 /* Use minimum usize to determine whether promotion may happen. */
1646 if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <=
1647 SMALL_MAXCLASS) {
1648 if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1649 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1),
1650 alignment, zero))
1651 return (old_usize);
1652 usize = isalloc(ptr, config_prof);
1653 if (max_usize < PAGE)
1654 arena_prof_promoted(ptr, usize);
1655 } else {
1656 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1657 zero, arena);
1658 }
1659
1660 return (usize);
1661 }
1662
1663 JEMALLOC_ALWAYS_INLINE_C size_t
1664 ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
1665 size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1666 prof_thr_cnt_t *cnt)
1667 {
1668 size_t usize;
1669 prof_ctx_t *old_ctx;
1670
1671 old_ctx = prof_ctx_get(ptr);
1672 if ((uintptr_t)cnt != (uintptr_t)1U) {
1673 usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
1674 alignment, zero, max_usize, arena, cnt);
1675 } else {
1676 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1677 zero, arena);
1678 }
1679 if (usize == old_usize)
1680 return (usize);
1681 prof_realloc(ptr, usize, cnt, old_usize, old_ctx);
1682
1683 return (usize);
1684 }
1685
1686 size_t
1687 je_xallocx(void *ptr, size_t size, size_t extra, int flags)
1688 {
1689 size_t usize, old_usize;
1690 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1691 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1692 & (SIZE_T_MAX-1));
1693 bool zero = flags & MALLOCX_ZERO;
1694 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1695 arena_t *arena;
1696
1697 assert(ptr != NULL);
1698 assert(size != 0);
1699 assert(SIZE_T_MAX - size >= extra);
1700 assert(malloc_initialized || IS_INITIALIZER);
1701 malloc_thread_init();
1702
1703 if (arena_ind != UINT_MAX)
1704 arena = arenas[arena_ind];
1705 else
1706 arena = NULL;
1707
1708 old_usize = isalloc(ptr, config_prof);
1709 if (config_valgrind && in_valgrind)
1710 old_rzsize = u2rz(old_usize);
1711
1712 if (config_prof && opt_prof) {
1713 prof_thr_cnt_t *cnt;
1714 /*
1715 * usize isn't knowable before ixalloc() returns when extra is
1716 * non-zero. Therefore, compute its maximum possible value and
1717 * use that in PROF_ALLOC_PREP() to decide whether to capture a
1718 * backtrace. prof_realloc() will use the actual usize to
1719 * decide whether to sample.
1720 */
1721 size_t max_usize = (alignment == 0) ? s2u(size+extra) :
1722 sa2u(size+extra, alignment);
1723 PROF_ALLOC_PREP(max_usize, cnt);
1724 usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
1725 max_usize, zero, arena, cnt);
1726 } else {
1727 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1728 zero, arena);
1729 }
1730 if (usize == old_usize)
1731 goto label_not_resized;
1732
1733 if (config_stats) {
1734 thread_allocated_t *ta;
1735 ta = thread_allocated_tsd_get();
1736 ta->allocated += usize;
1737 ta->deallocated += old_usize;
1738 }
1739 JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize,
1740 old_rzsize, false, zero);
1741 label_not_resized:
1742 UTRACE(ptr, size, ptr);
1743 return (usize);
1744 }
1745
1746 size_t
1747 je_sallocx(const void *ptr, int flags)
1748 {
1749 size_t usize;
1750
1751 assert(malloc_initialized || IS_INITIALIZER);
1752 malloc_thread_init();
1753
1754 if (config_ivsalloc)
1755 usize = ivsalloc(ptr, config_prof);
1756 else {
1757 assert(ptr != NULL);
1758 usize = isalloc(ptr, config_prof);
1759 }
1760
1761 return (usize);
1762 }
1763
1764 void
1765 je_dallocx(void *ptr, int flags)
1766 {
1767 size_t usize;
1768 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1769 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1770 bool try_tcache;
1771
1772 assert(ptr != NULL);
1773 assert(malloc_initialized || IS_INITIALIZER);
1774
1775 if (arena_ind != UINT_MAX) {
1776 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1777 try_tcache = (chunk == ptr || chunk->arena !=
1778 arenas[arena_ind]);
1779 } else
1780 try_tcache = true;
1781
1782 UTRACE(ptr, 0, 0);
1783 if (config_stats || config_valgrind)
1784 usize = isalloc(ptr, config_prof);
1785 if (config_prof && opt_prof) {
1786 if (config_stats == false && config_valgrind == false)
1787 usize = isalloc(ptr, config_prof);
1788 prof_free(ptr, usize);
1789 }
1790 if (config_stats)
1791 thread_allocated_tsd_get()->deallocated += usize;
1792 if (config_valgrind && in_valgrind)
1793 rzsize = p2rz(ptr);
1794 iqalloct(ptr, try_tcache);
1795 JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1796 }
1797
1798 size_t
1799 je_nallocx(size_t size, int flags)
1800 {
1801 size_t usize;
1802 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1803 & (SIZE_T_MAX-1));
1804
1805 assert(size != 0);
1806
1807 if (malloc_init())
1808 return (0);
1809
1810 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1811 assert(usize != 0);
1812 return (usize);
1813 }
1814
1815 int
1816 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
1817 size_t newlen)
1818 {
1819
1820 if (malloc_init())
1821 return (EAGAIN);
1822
1823 return (ctl_byname(name, oldp, oldlenp, newp, newlen));
1824 }
1825
1826 int
1827 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
1828 {
1829
1830 if (malloc_init())
1831 return (EAGAIN);
1832
1833 return (ctl_nametomib(name, mibp, miblenp));
1834 }
1835
1836 int
1837 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
1838 void *newp, size_t newlen)
1839 {
1840
1841 if (malloc_init())
1842 return (EAGAIN);
1843
1844 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
1845 }
1846
1847 void
1848 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
1849 const char *opts)
1850 {
1851
1852 stats_print(write_cb, cbopaque, opts);
1853 }
1854
1855 size_t
1856 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
1857 {
1858 size_t ret;
1859
1860 assert(malloc_initialized || IS_INITIALIZER);
1861 malloc_thread_init();
1862
1863 if (config_ivsalloc)
1864 ret = ivsalloc(ptr, config_prof);
1865 else
1866 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
1867
1868 return (ret);
1869 }
1870
1871 /*
1872 * End non-standard functions.
1873 */
1874 /******************************************************************************/
1875 /*
1876 * The following functions are used by threading libraries for protection of
1877 * malloc during fork().
1878 */
1879
1880 /*
1881 * If an application creates a thread before doing any allocation in the main
1882 * thread, then calls fork(2) in the main thread followed by memory allocation
1883 * in the child process, a race can occur that results in deadlock within the
1884 * child: the main thread may have forked while the created thread had
1885 * partially initialized the allocator. Ordinarily jemalloc prevents
1886 * fork/malloc races via the following functions it registers during
1887 * initialization using pthread_atfork(), but of course that does no good if
1888 * the allocator isn't fully initialized at fork time. The following library
1889 * constructor is a partial solution to this problem. It may still possible to
1890 * trigger the deadlock described above, but doing so would involve forking via
1891 * a library constructor that runs before jemalloc's runs.
1892 */
1893 JEMALLOC_ATTR(constructor)
1894 static void
1895 jemalloc_constructor(void)
1896 {
1897
1898 malloc_init();
1899 }
1900
1901 #ifndef JEMALLOC_MUTEX_INIT_CB
1902 void
1903 jemalloc_prefork(void)
1904 #else
1905 JEMALLOC_EXPORT void
1906 _malloc_prefork(void)
1907 #endif
1908 {
1909 unsigned i;
1910
1911 #ifdef JEMALLOC_MUTEX_INIT_CB
1912 if (malloc_initialized == false)
1913 return;
1914 #endif
1915 assert(malloc_initialized);
1916
1917 /* Acquire all mutexes in a safe order. */
1918 ctl_prefork();
1919 prof_prefork();
1920 malloc_mutex_prefork(&arenas_lock);
1921 for (i = 0; i < narenas_total; i++) {
1922 if (arenas[i] != NULL)
1923 arena_prefork(arenas[i]);
1924 }
1925 chunk_prefork();
1926 base_prefork();
1927 huge_prefork();
1928 }
1929
1930 #ifndef JEMALLOC_MUTEX_INIT_CB
1931 void
1932 jemalloc_postfork_parent(void)
1933 #else
1934 JEMALLOC_EXPORT void
1935 _malloc_postfork(void)
1936 #endif
1937 {
1938 unsigned i;
1939
1940 #ifdef JEMALLOC_MUTEX_INIT_CB
1941 if (malloc_initialized == false)
1942 return;
1943 #endif
1944 assert(malloc_initialized);
1945
1946 /* Release all mutexes, now that fork() has completed. */
1947 huge_postfork_parent();
1948 base_postfork_parent();
1949 chunk_postfork_parent();
1950 for (i = 0; i < narenas_total; i++) {
1951 if (arenas[i] != NULL)
1952 arena_postfork_parent(arenas[i]);
1953 }
1954 malloc_mutex_postfork_parent(&arenas_lock);
1955 prof_postfork_parent();
1956 ctl_postfork_parent();
1957 }
1958
1959 void
1960 jemalloc_postfork_child(void)
1961 {
1962 unsigned i;
1963
1964 assert(malloc_initialized);
1965
1966 /* Release all mutexes, now that fork() has completed. */
1967 huge_postfork_child();
1968 base_postfork_child();
1969 chunk_postfork_child();
1970 for (i = 0; i < narenas_total; i++) {
1971 if (arenas[i] != NULL)
1972 arena_postfork_child(arenas[i]);
1973 }
1974 malloc_mutex_postfork_child(&arenas_lock);
1975 prof_postfork_child();
1976 ctl_postfork_child();
1977 }
1978
1979 /******************************************************************************/
1980 /*
1981 * The following functions are used for TLS allocation/deallocation in static
1982 * binaries on FreeBSD. The primary difference between these and i[mcd]alloc()
1983 * is that these avoid accessing TLS variables.
1984 */
1985
1986 static void *
1987 a0alloc(size_t size, bool zero)
1988 {
1989
1990 if (malloc_init())
1991 return (NULL);
1992
1993 if (size == 0)
1994 size = 1;
1995
1996 if (size <= arena_maxclass)
1997 return (arena_malloc(arenas[0], size, zero, false));
1998 else
1999 return (huge_malloc(NULL, size, zero));
2000 }
2001
2002 void *
2003 a0malloc(size_t size)
2004 {
2005
2006 return (a0alloc(size, false));
2007 }
2008
2009 void *
2010 a0calloc(size_t num, size_t size)
2011 {
2012
2013 return (a0alloc(num * size, true));
2014 }
2015
2016 void
2017 a0free(void *ptr)
2018 {
2019 arena_chunk_t *chunk;
2020
2021 if (ptr == NULL)
2022 return;
2023
2024 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2025 if (chunk != ptr)
2026 arena_dalloc(chunk, ptr, false);
2027 else
2028 huge_dalloc(ptr);
2029 }
2030
2031 /******************************************************************************/
2032