1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 /* Maximum number of malloc_tsd users with cleanup functions. */
5 #define MALLOC_TSD_CLEANUPS_MAX 2
6
7 typedef bool (*malloc_tsd_cleanup_t)(void);
8
9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
10 !defined(_WIN32))
11 typedef struct tsd_init_block_s tsd_init_block_t;
12 typedef struct tsd_init_head_s tsd_init_head_t;
13 #endif
14
15 typedef struct tsd_s tsd_t;
16 typedef struct tsdn_s tsdn_t;
17
18 #define TSDN_NULL ((tsdn_t *)0)
19
20 typedef enum {
21 tsd_state_uninitialized,
22 tsd_state_nominal,
23 tsd_state_purgatory,
24 tsd_state_reincarnated
25 } tsd_state_t;
26
27 /*
28 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There
29 * are five macros that support (at least) three use cases: file-private,
30 * library-private, and library-private inlined. Following is an example
31 * library-private tsd variable:
32 *
33 * In example.h:
34 * typedef struct {
35 * int x;
36 * int y;
37 * } example_t;
38 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
39 * malloc_tsd_types(example_, example_t)
40 * malloc_tsd_protos(, example_, example_t)
41 * malloc_tsd_externs(example_, example_t)
42 * In example.c:
43 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
44 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
45 * example_tsd_cleanup)
46 *
47 * The result is a set of generated functions, e.g.:
48 *
49 * bool example_tsd_boot(void) {...}
50 * bool example_tsd_booted_get(void) {...}
51 * example_t *example_tsd_get(bool init) {...}
52 * void example_tsd_set(example_t *val) {...}
53 *
54 * Note that all of the functions deal in terms of (a_type *) rather than
55 * (a_type) so that it is possible to support non-pointer types (unlike
56 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is
57 * cast to (void *). This means that the cleanup function needs to cast the
58 * function argument to (a_type *), then dereference the resulting pointer to
59 * access fields, e.g.
60 *
61 * void
62 * example_tsd_cleanup(void *arg)
63 * {
64 * example_t *example = (example_t *)arg;
65 *
66 * example->x = 42;
67 * [...]
68 * if ([want the cleanup function to be called again])
69 * example_tsd_set(example);
70 * }
71 *
72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be
73 * called again. This is similar to how pthreads TSD destruction works, except
74 * that pthreads only calls the cleanup function again if the value was set to
75 * non-NULL.
76 */
77
78 /* malloc_tsd_types(). */
79 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
80 #define malloc_tsd_types(a_name, a_type)
81 #elif (defined(JEMALLOC_TLS))
82 #define malloc_tsd_types(a_name, a_type)
83 #elif (defined(_WIN32))
84 #define malloc_tsd_types(a_name, a_type) \
85 typedef struct { \
86 bool initialized; \
87 a_type val; \
88 } a_name##tsd_wrapper_t;
89 #else
90 #define malloc_tsd_types(a_name, a_type) \
91 typedef struct { \
92 bool initialized; \
93 a_type val; \
94 } a_name##tsd_wrapper_t;
95 #endif
96
97 /* malloc_tsd_protos(). */
98 #define malloc_tsd_protos(a_attr, a_name, a_type) \
99 a_attr bool \
100 a_name##tsd_boot0(void); \
101 a_attr void \
102 a_name##tsd_boot1(void); \
103 a_attr bool \
104 a_name##tsd_boot(void); \
105 a_attr bool \
106 a_name##tsd_booted_get(void); \
107 a_attr a_type * \
108 a_name##tsd_get(bool init); \
109 a_attr void \
110 a_name##tsd_set(a_type *val);
111
112 /* malloc_tsd_externs(). */
113 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
114 #define malloc_tsd_externs(a_name, a_type) \
115 extern __thread a_type a_name##tsd_tls; \
116 extern __thread bool a_name##tsd_initialized; \
117 extern bool a_name##tsd_booted;
118 #elif (defined(JEMALLOC_TLS))
119 #define malloc_tsd_externs(a_name, a_type) \
120 extern __thread a_type a_name##tsd_tls; \
121 extern pthread_key_t a_name##tsd_tsd; \
122 extern bool a_name##tsd_booted;
123 #elif (defined(_WIN32))
124 #define malloc_tsd_externs(a_name, a_type) \
125 extern DWORD a_name##tsd_tsd; \
126 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
127 extern bool a_name##tsd_booted;
128 #else
129 #define malloc_tsd_externs(a_name, a_type) \
130 extern pthread_key_t a_name##tsd_tsd; \
131 extern tsd_init_head_t a_name##tsd_init_head; \
132 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
133 extern bool a_name##tsd_booted;
134 #endif
135
136 /* malloc_tsd_data(). */
137 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
138 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
139 a_attr __thread a_type JEMALLOC_TLS_MODEL \
140 a_name##tsd_tls = a_initializer; \
141 a_attr __thread bool JEMALLOC_TLS_MODEL \
142 a_name##tsd_initialized = false; \
143 a_attr bool a_name##tsd_booted = false;
144 #elif (defined(JEMALLOC_TLS))
145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
146 a_attr __thread a_type JEMALLOC_TLS_MODEL \
147 a_name##tsd_tls = a_initializer; \
148 a_attr pthread_key_t a_name##tsd_tsd; \
149 a_attr bool a_name##tsd_booted = false;
150 #elif (defined(_WIN32))
151 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
152 a_attr DWORD a_name##tsd_tsd; \
153 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
154 false, \
155 a_initializer \
156 }; \
157 a_attr bool a_name##tsd_booted = false;
158 #else
159 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
160 a_attr pthread_key_t a_name##tsd_tsd; \
161 a_attr tsd_init_head_t a_name##tsd_init_head = { \
162 ql_head_initializer(blocks), \
163 MALLOC_MUTEX_INITIALIZER \
164 }; \
165 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
166 false, \
167 a_initializer \
168 }; \
169 a_attr bool a_name##tsd_booted = false;
170 #endif
171
172 /* malloc_tsd_funcs(). */
173 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
174 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
175 a_cleanup) \
176 /* Initialization/cleanup. */ \
177 a_attr bool \
178 a_name##tsd_cleanup_wrapper(void) \
179 { \
180 \
181 if (a_name##tsd_initialized) { \
182 a_name##tsd_initialized = false; \
183 a_cleanup(&a_name##tsd_tls); \
184 } \
185 return (a_name##tsd_initialized); \
186 } \
187 a_attr bool \
188 a_name##tsd_boot0(void) \
189 { \
190 \
191 if (a_cleanup != malloc_tsd_no_cleanup) { \
192 malloc_tsd_cleanup_register( \
193 &a_name##tsd_cleanup_wrapper); \
194 } \
195 a_name##tsd_booted = true; \
196 return (false); \
197 } \
198 a_attr void \
199 a_name##tsd_boot1(void) \
200 { \
201 \
202 /* Do nothing. */ \
203 } \
204 a_attr bool \
205 a_name##tsd_boot(void) \
206 { \
207 \
208 return (a_name##tsd_boot0()); \
209 } \
210 a_attr bool \
211 a_name##tsd_booted_get(void) \
212 { \
213 \
214 return (a_name##tsd_booted); \
215 } \
216 a_attr bool \
217 a_name##tsd_get_allocates(void) \
218 { \
219 \
220 return (false); \
221 } \
222 /* Get/set. */ \
223 a_attr a_type * \
224 a_name##tsd_get(bool init) \
225 { \
226 \
227 assert(a_name##tsd_booted); \
228 return (&a_name##tsd_tls); \
229 } \
230 a_attr void \
231 a_name##tsd_set(a_type *val) \
232 { \
233 \
234 assert(a_name##tsd_booted); \
235 a_name##tsd_tls = (*val); \
236 if (a_cleanup != malloc_tsd_no_cleanup) \
237 a_name##tsd_initialized = true; \
238 }
239 #elif (defined(JEMALLOC_TLS))
240 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
241 a_cleanup) \
242 /* Initialization/cleanup. */ \
243 a_attr bool \
244 a_name##tsd_boot0(void) \
245 { \
246 \
247 if (a_cleanup != malloc_tsd_no_cleanup) { \
248 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
249 0) \
250 return (true); \
251 } \
252 a_name##tsd_booted = true; \
253 return (false); \
254 } \
255 a_attr void \
256 a_name##tsd_boot1(void) \
257 { \
258 \
259 /* Do nothing. */ \
260 } \
261 a_attr bool \
262 a_name##tsd_boot(void) \
263 { \
264 \
265 return (a_name##tsd_boot0()); \
266 } \
267 a_attr bool \
268 a_name##tsd_booted_get(void) \
269 { \
270 \
271 return (a_name##tsd_booted); \
272 } \
273 a_attr bool \
274 a_name##tsd_get_allocates(void) \
275 { \
276 \
277 return (false); \
278 } \
279 /* Get/set. */ \
280 a_attr a_type * \
281 a_name##tsd_get(bool init) \
282 { \
283 \
284 assert(a_name##tsd_booted); \
285 return (&a_name##tsd_tls); \
286 } \
287 a_attr void \
288 a_name##tsd_set(a_type *val) \
289 { \
290 \
291 assert(a_name##tsd_booted); \
292 a_name##tsd_tls = (*val); \
293 if (a_cleanup != malloc_tsd_no_cleanup) { \
294 if (pthread_setspecific(a_name##tsd_tsd, \
295 (void *)(&a_name##tsd_tls))) { \
296 malloc_write("<jemalloc>: Error" \
297 " setting TSD for "#a_name"\n"); \
298 if (opt_abort) \
299 abort(); \
300 } \
301 } \
302 }
303 #elif (defined(_WIN32))
304 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
305 a_cleanup) \
306 /* Initialization/cleanup. */ \
307 a_attr bool \
308 a_name##tsd_cleanup_wrapper(void) \
309 { \
310 DWORD error = GetLastError(); \
311 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
312 TlsGetValue(a_name##tsd_tsd); \
313 SetLastError(error); \
314 \
315 if (wrapper == NULL) \
316 return (false); \
317 if (a_cleanup != malloc_tsd_no_cleanup && \
318 wrapper->initialized) { \
319 wrapper->initialized = false; \
320 a_cleanup(&wrapper->val); \
321 if (wrapper->initialized) { \
322 /* Trigger another cleanup round. */ \
323 return (true); \
324 } \
325 } \
326 malloc_tsd_dalloc(wrapper); \
327 return (false); \
328 } \
329 a_attr void \
330 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
331 { \
332 \
333 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
334 malloc_write("<jemalloc>: Error setting" \
335 " TSD for "#a_name"\n"); \
336 abort(); \
337 } \
338 } \
339 a_attr a_name##tsd_wrapper_t * \
340 a_name##tsd_wrapper_get(bool init) \
341 { \
342 DWORD error = GetLastError(); \
343 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
344 TlsGetValue(a_name##tsd_tsd); \
345 SetLastError(error); \
346 \
347 if (init && unlikely(wrapper == NULL)) { \
348 wrapper = (a_name##tsd_wrapper_t *) \
349 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
350 if (wrapper == NULL) { \
351 malloc_write("<jemalloc>: Error allocating" \
352 " TSD for "#a_name"\n"); \
353 abort(); \
354 } else { \
355 wrapper->initialized = false; \
356 wrapper->val = a_initializer; \
357 } \
358 a_name##tsd_wrapper_set(wrapper); \
359 } \
360 return (wrapper); \
361 } \
362 a_attr bool \
363 a_name##tsd_boot0(void) \
364 { \
365 \
366 a_name##tsd_tsd = TlsAlloc(); \
367 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
368 return (true); \
369 if (a_cleanup != malloc_tsd_no_cleanup) { \
370 malloc_tsd_cleanup_register( \
371 &a_name##tsd_cleanup_wrapper); \
372 } \
373 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
374 a_name##tsd_booted = true; \
375 return (false); \
376 } \
377 a_attr void \
378 a_name##tsd_boot1(void) \
379 { \
380 a_name##tsd_wrapper_t *wrapper; \
381 wrapper = (a_name##tsd_wrapper_t *) \
382 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
383 if (wrapper == NULL) { \
384 malloc_write("<jemalloc>: Error allocating" \
385 " TSD for "#a_name"\n"); \
386 abort(); \
387 } \
388 memcpy(wrapper, &a_name##tsd_boot_wrapper, \
389 sizeof(a_name##tsd_wrapper_t)); \
390 a_name##tsd_wrapper_set(wrapper); \
391 } \
392 a_attr bool \
393 a_name##tsd_boot(void) \
394 { \
395 \
396 if (a_name##tsd_boot0()) \
397 return (true); \
398 a_name##tsd_boot1(); \
399 return (false); \
400 } \
401 a_attr bool \
402 a_name##tsd_booted_get(void) \
403 { \
404 \
405 return (a_name##tsd_booted); \
406 } \
407 a_attr bool \
408 a_name##tsd_get_allocates(void) \
409 { \
410 \
411 return (true); \
412 } \
413 /* Get/set. */ \
414 a_attr a_type * \
415 a_name##tsd_get(bool init) \
416 { \
417 a_name##tsd_wrapper_t *wrapper; \
418 \
419 assert(a_name##tsd_booted); \
420 wrapper = a_name##tsd_wrapper_get(init); \
421 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
422 return (NULL); \
423 return (&wrapper->val); \
424 } \
425 a_attr void \
426 a_name##tsd_set(a_type *val) \
427 { \
428 a_name##tsd_wrapper_t *wrapper; \
429 \
430 assert(a_name##tsd_booted); \
431 wrapper = a_name##tsd_wrapper_get(true); \
432 wrapper->val = *(val); \
433 if (a_cleanup != malloc_tsd_no_cleanup) \
434 wrapper->initialized = true; \
435 }
436 #else
437 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
438 a_cleanup) \
439 /* Initialization/cleanup. */ \
440 a_attr void \
441 a_name##tsd_cleanup_wrapper(void *arg) \
442 { \
443 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
444 \
445 if (a_cleanup != malloc_tsd_no_cleanup && \
446 wrapper->initialized) { \
447 wrapper->initialized = false; \
448 a_cleanup(&wrapper->val); \
449 if (wrapper->initialized) { \
450 /* Trigger another cleanup round. */ \
451 if (pthread_setspecific(a_name##tsd_tsd, \
452 (void *)wrapper)) { \
453 malloc_write("<jemalloc>: Error" \
454 " setting TSD for "#a_name"\n"); \
455 if (opt_abort) \
456 abort(); \
457 } \
458 return; \
459 } \
460 } \
461 malloc_tsd_dalloc(wrapper); \
462 } \
463 a_attr void \
464 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
465 { \
466 \
467 if (pthread_setspecific(a_name##tsd_tsd, \
468 (void *)wrapper)) { \
469 malloc_write("<jemalloc>: Error setting" \
470 " TSD for "#a_name"\n"); \
471 abort(); \
472 } \
473 } \
474 a_attr a_name##tsd_wrapper_t * \
475 a_name##tsd_wrapper_get(bool init) \
476 { \
477 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
478 pthread_getspecific(a_name##tsd_tsd); \
479 \
480 if (init && unlikely(wrapper == NULL)) { \
481 tsd_init_block_t block; \
482 wrapper = tsd_init_check_recursion( \
483 &a_name##tsd_init_head, &block); \
484 if (wrapper) \
485 return (wrapper); \
486 wrapper = (a_name##tsd_wrapper_t *) \
487 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
488 block.data = wrapper; \
489 if (wrapper == NULL) { \
490 malloc_write("<jemalloc>: Error allocating" \
491 " TSD for "#a_name"\n"); \
492 abort(); \
493 } else { \
494 wrapper->initialized = false; \
495 wrapper->val = a_initializer; \
496 } \
497 a_name##tsd_wrapper_set(wrapper); \
498 tsd_init_finish(&a_name##tsd_init_head, &block); \
499 } \
500 return (wrapper); \
501 } \
502 a_attr bool \
503 a_name##tsd_boot0(void) \
504 { \
505 \
506 if (pthread_key_create(&a_name##tsd_tsd, \
507 a_name##tsd_cleanup_wrapper) != 0) \
508 return (true); \
509 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
510 a_name##tsd_booted = true; \
511 return (false); \
512 } \
513 a_attr void \
514 a_name##tsd_boot1(void) \
515 { \
516 a_name##tsd_wrapper_t *wrapper; \
517 wrapper = (a_name##tsd_wrapper_t *) \
518 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
519 if (wrapper == NULL) { \
520 malloc_write("<jemalloc>: Error allocating" \
521 " TSD for "#a_name"\n"); \
522 abort(); \
523 } \
524 memcpy(wrapper, &a_name##tsd_boot_wrapper, \
525 sizeof(a_name##tsd_wrapper_t)); \
526 a_name##tsd_wrapper_set(wrapper); \
527 } \
528 a_attr bool \
529 a_name##tsd_boot(void) \
530 { \
531 \
532 if (a_name##tsd_boot0()) \
533 return (true); \
534 a_name##tsd_boot1(); \
535 return (false); \
536 } \
537 a_attr bool \
538 a_name##tsd_booted_get(void) \
539 { \
540 \
541 return (a_name##tsd_booted); \
542 } \
543 a_attr bool \
544 a_name##tsd_get_allocates(void) \
545 { \
546 \
547 return (true); \
548 } \
549 /* Get/set. */ \
550 a_attr a_type * \
551 a_name##tsd_get(bool init) \
552 { \
553 a_name##tsd_wrapper_t *wrapper; \
554 \
555 assert(a_name##tsd_booted); \
556 wrapper = a_name##tsd_wrapper_get(init); \
557 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
558 return (NULL); \
559 return (&wrapper->val); \
560 } \
561 a_attr void \
562 a_name##tsd_set(a_type *val) \
563 { \
564 a_name##tsd_wrapper_t *wrapper; \
565 \
566 assert(a_name##tsd_booted); \
567 wrapper = a_name##tsd_wrapper_get(true); \
568 wrapper->val = *(val); \
569 if (a_cleanup != malloc_tsd_no_cleanup) \
570 wrapper->initialized = true; \
571 }
572 #endif
573
574 #endif /* JEMALLOC_H_TYPES */
575 /******************************************************************************/
576 #ifdef JEMALLOC_H_STRUCTS
577
578 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
579 !defined(_WIN32))
580 struct tsd_init_block_s {
581 ql_elm(tsd_init_block_t) link;
582 pthread_t thread;
583 void *data;
584 };
585 struct tsd_init_head_s {
586 ql_head(tsd_init_block_t) blocks;
587 malloc_mutex_t lock;
588 };
589 #endif
590
591 #define MALLOC_TSD \
592 /* O(name, type) */ \
593 O(tcache, tcache_t *) \
594 O(thread_allocated, uint64_t) \
595 O(thread_deallocated, uint64_t) \
596 O(prof_tdata, prof_tdata_t *) \
597 O(iarena, arena_t *) \
598 O(arena, arena_t *) \
599 O(arenas_tdata, arena_tdata_t *) \
600 O(narenas_tdata, unsigned) \
601 O(arenas_tdata_bypass, bool) \
602 O(tcache_enabled, tcache_enabled_t) \
603 O(quarantine, quarantine_t *) \
604 O(witnesses, witness_list_t) \
605 O(witness_fork, bool) \
606
607 #define TSD_INITIALIZER { \
608 tsd_state_uninitialized, \
609 NULL, \
610 0, \
611 0, \
612 NULL, \
613 NULL, \
614 NULL, \
615 NULL, \
616 0, \
617 false, \
618 tcache_enabled_default, \
619 NULL, \
620 ql_head_initializer(witnesses), \
621 false \
622 }
623
624 struct tsd_s {
625 tsd_state_t state;
626 #define O(n, t) \
627 t n;
628 MALLOC_TSD
629 #undef O
630 };
631
632 /*
633 * Wrapper around tsd_t that makes it possible to avoid implicit conversion
634 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be
635 * explicitly converted to tsd_t, which is non-nullable.
636 */
637 struct tsdn_s {
638 tsd_t tsd;
639 };
640
641 static const tsd_t tsd_initializer = TSD_INITIALIZER;
642
643 malloc_tsd_types(, tsd_t)
644
645 #endif /* JEMALLOC_H_STRUCTS */
646 /******************************************************************************/
647 #ifdef JEMALLOC_H_EXTERNS
648
649 void *malloc_tsd_malloc(size_t size);
650 void malloc_tsd_dalloc(void *wrapper);
651 void malloc_tsd_no_cleanup(void *arg);
652 void malloc_tsd_cleanup_register(bool (*f)(void));
653 tsd_t *malloc_tsd_boot0(void);
654 void malloc_tsd_boot1(void);
655 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
656 !defined(_WIN32))
657 void *tsd_init_check_recursion(tsd_init_head_t *head,
658 tsd_init_block_t *block);
659 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
660 #endif
661 void tsd_cleanup(void *arg);
662
663 #endif /* JEMALLOC_H_EXTERNS */
664 /******************************************************************************/
665 #ifdef JEMALLOC_H_INLINES
666
667 #ifndef JEMALLOC_ENABLE_INLINE
668 malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
669
670 tsd_t *tsd_fetch_impl(bool init);
671 tsd_t *tsd_fetch(void);
672 tsdn_t *tsd_tsdn(tsd_t *tsd);
673 bool tsd_nominal(tsd_t *tsd);
674 #define O(n, t) \
675 t *tsd_##n##p_get(tsd_t *tsd); \
676 t tsd_##n##_get(tsd_t *tsd); \
677 void tsd_##n##_set(tsd_t *tsd, t n);
678 MALLOC_TSD
679 #undef O
680 tsdn_t *tsdn_fetch(void);
681 bool tsdn_null(const tsdn_t *tsdn);
682 tsd_t *tsdn_tsd(tsdn_t *tsdn);
683 #endif
684
685 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
686 malloc_tsd_externs(, tsd_t)
687 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
688
689 JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_fetch_impl(bool init)690 tsd_fetch_impl(bool init)
691 {
692 tsd_t *tsd = tsd_get(init);
693
694 if (!init && tsd_get_allocates() && tsd == NULL)
695 return (NULL);
696 assert(tsd != NULL);
697
698 if (unlikely(tsd->state != tsd_state_nominal)) {
699 if (tsd->state == tsd_state_uninitialized) {
700 tsd->state = tsd_state_nominal;
701 /* Trigger cleanup handler registration. */
702 tsd_set(tsd);
703 } else if (tsd->state == tsd_state_purgatory) {
704 tsd->state = tsd_state_reincarnated;
705 tsd_set(tsd);
706 } else
707 assert(tsd->state == tsd_state_reincarnated);
708 }
709
710 return (tsd);
711 }
712
713 JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_fetch(void)714 tsd_fetch(void)
715 {
716
717 return (tsd_fetch_impl(true));
718 }
719
720 JEMALLOC_ALWAYS_INLINE tsdn_t *
tsd_tsdn(tsd_t * tsd)721 tsd_tsdn(tsd_t *tsd)
722 {
723
724 return ((tsdn_t *)tsd);
725 }
726
727 JEMALLOC_INLINE bool
tsd_nominal(tsd_t * tsd)728 tsd_nominal(tsd_t *tsd)
729 {
730
731 return (tsd->state == tsd_state_nominal);
732 }
733
734 #define O(n, t) \
735 JEMALLOC_ALWAYS_INLINE t * \
736 tsd_##n##p_get(tsd_t *tsd) \
737 { \
738 \
739 return (&tsd->n); \
740 } \
741 \
742 JEMALLOC_ALWAYS_INLINE t \
743 tsd_##n##_get(tsd_t *tsd) \
744 { \
745 \
746 return (*tsd_##n##p_get(tsd)); \
747 } \
748 \
749 JEMALLOC_ALWAYS_INLINE void \
750 tsd_##n##_set(tsd_t *tsd, t n) \
751 { \
752 \
753 assert(tsd->state == tsd_state_nominal); \
754 tsd->n = n; \
755 }
756 MALLOC_TSD
757 #undef O
758
759 JEMALLOC_ALWAYS_INLINE tsdn_t *
tsdn_fetch(void)760 tsdn_fetch(void)
761 {
762
763 if (!tsd_booted_get())
764 return (NULL);
765
766 return (tsd_tsdn(tsd_fetch_impl(false)));
767 }
768
769 JEMALLOC_ALWAYS_INLINE bool
tsdn_null(const tsdn_t * tsdn)770 tsdn_null(const tsdn_t *tsdn)
771 {
772
773 return (tsdn == NULL);
774 }
775
776 JEMALLOC_ALWAYS_INLINE tsd_t *
tsdn_tsd(tsdn_t * tsdn)777 tsdn_tsd(tsdn_t *tsdn)
778 {
779
780 assert(!tsdn_null(tsdn));
781
782 return (&tsdn->tsd);
783 }
784 #endif
785
786 #endif /* JEMALLOC_H_INLINES */
787 /******************************************************************************/
788