1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 /* Maximum number of malloc_tsd users with cleanup functions. */ 5 #define MALLOC_TSD_CLEANUPS_MAX 8 6 7 typedef bool (*malloc_tsd_cleanup_t)(void); 8 9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10 !defined(_WIN32)) 11 typedef struct tsd_init_block_s tsd_init_block_t; 12 typedef struct tsd_init_head_s tsd_init_head_t; 13 #endif 14 15 /* 16 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 17 * are four macros that support (at least) three use cases: file-private, 18 * library-private, and library-private inlined. Following is an example 19 * library-private tsd variable: 20 * 21 * In example.h: 22 * typedef struct { 23 * int x; 24 * int y; 25 * } example_t; 26 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 27 * malloc_tsd_protos(, example, example_t *) 28 * malloc_tsd_externs(example, example_t *) 29 * In example.c: 30 * malloc_tsd_data(, example, example_t *, EX_INITIALIZER) 31 * malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER, 32 * example_tsd_cleanup) 33 * 34 * The result is a set of generated functions, e.g.: 35 * 36 * bool example_tsd_boot(void) {...} 37 * example_t **example_tsd_get() {...} 38 * void example_tsd_set(example_t **val) {...} 39 * 40 * Note that all of the functions deal in terms of (a_type *) rather than 41 * (a_type) so that it is possible to support non-pointer types (unlike 42 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 43 * cast to (void *). This means that the cleanup function needs to cast *and* 44 * dereference the function argument, e.g.: 45 * 46 * void 47 * example_tsd_cleanup(void *arg) 48 * { 49 * example_t *example = *(example_t **)arg; 50 * 51 * [...] 52 * if ([want the cleanup function to be called again]) { 53 * example_tsd_set(&example); 54 * } 55 * } 56 * 57 * If example_tsd_set() is called within example_tsd_cleanup(), it will be 58 * called again. This is similar to how pthreads TSD destruction works, except 59 * that pthreads only calls the cleanup function again if the value was set to 60 * non-NULL. 61 */ 62 63 /* malloc_tsd_protos(). */ 64 #define malloc_tsd_protos(a_attr, a_name, a_type) \ 65 a_attr bool \ 66 a_name##_tsd_boot(void); \ 67 a_attr a_type * \ 68 a_name##_tsd_get(void); \ 69 a_attr void \ 70 a_name##_tsd_set(a_type *val); 71 72 /* malloc_tsd_externs(). */ 73 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 74 #define malloc_tsd_externs(a_name, a_type) \ 75 extern __thread a_type a_name##_tls; \ 76 extern __thread bool a_name##_initialized; \ 77 extern bool a_name##_booted; 78 #elif (defined(JEMALLOC_TLS)) 79 #define malloc_tsd_externs(a_name, a_type) \ 80 extern __thread a_type a_name##_tls; \ 81 extern pthread_key_t a_name##_tsd; \ 82 extern bool a_name##_booted; 83 #elif (defined(_WIN32)) 84 #define malloc_tsd_externs(a_name, a_type) \ 85 extern DWORD a_name##_tsd; \ 86 extern bool a_name##_booted; 87 #else 88 #define malloc_tsd_externs(a_name, a_type) \ 89 extern pthread_key_t a_name##_tsd; \ 90 extern tsd_init_head_t a_name##_tsd_init_head; \ 91 extern bool a_name##_booted; 92 #endif 93 94 /* malloc_tsd_data(). */ 95 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 96 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 97 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 98 a_name##_tls = a_initializer; \ 99 a_attr __thread bool JEMALLOC_TLS_MODEL \ 100 a_name##_initialized = false; \ 101 a_attr bool a_name##_booted = false; 102 #elif (defined(JEMALLOC_TLS)) 103 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 104 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 105 a_name##_tls = a_initializer; \ 106 a_attr pthread_key_t a_name##_tsd; \ 107 a_attr bool a_name##_booted = false; 108 #elif (defined(_WIN32)) 109 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 110 a_attr DWORD a_name##_tsd; \ 111 a_attr bool a_name##_booted = false; 112 #else 113 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 114 a_attr pthread_key_t a_name##_tsd; \ 115 a_attr tsd_init_head_t a_name##_tsd_init_head = { \ 116 ql_head_initializer(blocks), \ 117 MALLOC_MUTEX_INITIALIZER \ 118 }; \ 119 a_attr bool a_name##_booted = false; 120 #endif 121 122 /* malloc_tsd_funcs(). */ 123 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 124 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 125 a_cleanup) \ 126 /* Initialization/cleanup. */ \ 127 a_attr bool \ 128 a_name##_tsd_cleanup_wrapper(void) \ 129 { \ 130 \ 131 if (a_name##_initialized) { \ 132 a_name##_initialized = false; \ 133 a_cleanup(&a_name##_tls); \ 134 } \ 135 return (a_name##_initialized); \ 136 } \ 137 a_attr bool \ 138 a_name##_tsd_boot(void) \ 139 { \ 140 \ 141 if (a_cleanup != malloc_tsd_no_cleanup) { \ 142 malloc_tsd_cleanup_register( \ 143 &a_name##_tsd_cleanup_wrapper); \ 144 } \ 145 a_name##_booted = true; \ 146 return (false); \ 147 } \ 148 /* Get/set. */ \ 149 a_attr a_type * \ 150 a_name##_tsd_get(void) \ 151 { \ 152 \ 153 assert(a_name##_booted); \ 154 return (&a_name##_tls); \ 155 } \ 156 a_attr void \ 157 a_name##_tsd_set(a_type *val) \ 158 { \ 159 \ 160 assert(a_name##_booted); \ 161 a_name##_tls = (*val); \ 162 if (a_cleanup != malloc_tsd_no_cleanup) \ 163 a_name##_initialized = true; \ 164 } 165 #elif (defined(JEMALLOC_TLS)) 166 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 167 a_cleanup) \ 168 /* Initialization/cleanup. */ \ 169 a_attr bool \ 170 a_name##_tsd_boot(void) \ 171 { \ 172 \ 173 if (a_cleanup != malloc_tsd_no_cleanup) { \ 174 if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \ 175 return (true); \ 176 } \ 177 a_name##_booted = true; \ 178 return (false); \ 179 } \ 180 /* Get/set. */ \ 181 a_attr a_type * \ 182 a_name##_tsd_get(void) \ 183 { \ 184 \ 185 assert(a_name##_booted); \ 186 return (&a_name##_tls); \ 187 } \ 188 a_attr void \ 189 a_name##_tsd_set(a_type *val) \ 190 { \ 191 \ 192 assert(a_name##_booted); \ 193 a_name##_tls = (*val); \ 194 if (a_cleanup != malloc_tsd_no_cleanup) { \ 195 if (pthread_setspecific(a_name##_tsd, \ 196 (void *)(&a_name##_tls))) { \ 197 malloc_write("<jemalloc>: Error" \ 198 " setting TSD for "#a_name"\n"); \ 199 if (opt_abort) \ 200 abort(); \ 201 } \ 202 } \ 203 } 204 #elif (defined(_WIN32)) 205 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 206 a_cleanup) \ 207 /* Data structure. */ \ 208 typedef struct { \ 209 bool initialized; \ 210 a_type val; \ 211 } a_name##_tsd_wrapper_t; \ 212 /* Initialization/cleanup. */ \ 213 a_attr bool \ 214 a_name##_tsd_cleanup_wrapper(void) \ 215 { \ 216 a_name##_tsd_wrapper_t *wrapper; \ 217 \ 218 wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \ 219 if (wrapper == NULL) \ 220 return (false); \ 221 if (a_cleanup != malloc_tsd_no_cleanup && \ 222 wrapper->initialized) { \ 223 a_type val = wrapper->val; \ 224 a_type tsd_static_data = a_initializer; \ 225 wrapper->initialized = false; \ 226 wrapper->val = tsd_static_data; \ 227 a_cleanup(&val); \ 228 if (wrapper->initialized) { \ 229 /* Trigger another cleanup round. */ \ 230 return (true); \ 231 } \ 232 } \ 233 malloc_tsd_dalloc(wrapper); \ 234 return (false); \ 235 } \ 236 a_attr bool \ 237 a_name##_tsd_boot(void) \ 238 { \ 239 \ 240 a_name##_tsd = TlsAlloc(); \ 241 if (a_name##_tsd == TLS_OUT_OF_INDEXES) \ 242 return (true); \ 243 if (a_cleanup != malloc_tsd_no_cleanup) { \ 244 malloc_tsd_cleanup_register( \ 245 &a_name##_tsd_cleanup_wrapper); \ 246 } \ 247 a_name##_booted = true; \ 248 return (false); \ 249 } \ 250 /* Get/set. */ \ 251 a_attr a_name##_tsd_wrapper_t * \ 252 a_name##_tsd_get_wrapper(void) \ 253 { \ 254 a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ 255 TlsGetValue(a_name##_tsd); \ 256 \ 257 if (wrapper == NULL) { \ 258 wrapper = (a_name##_tsd_wrapper_t *) \ 259 malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ 260 if (wrapper == NULL) { \ 261 malloc_write("<jemalloc>: Error allocating" \ 262 " TSD for "#a_name"\n"); \ 263 abort(); \ 264 } else { \ 265 static a_type tsd_static_data = a_initializer; \ 266 wrapper->initialized = false; \ 267 wrapper->val = tsd_static_data; \ 268 } \ 269 if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \ 270 malloc_write("<jemalloc>: Error setting" \ 271 " TSD for "#a_name"\n"); \ 272 abort(); \ 273 } \ 274 } \ 275 return (wrapper); \ 276 } \ 277 a_attr a_type * \ 278 a_name##_tsd_get(void) \ 279 { \ 280 a_name##_tsd_wrapper_t *wrapper; \ 281 \ 282 assert(a_name##_booted); \ 283 wrapper = a_name##_tsd_get_wrapper(); \ 284 return (&wrapper->val); \ 285 } \ 286 a_attr void \ 287 a_name##_tsd_set(a_type *val) \ 288 { \ 289 a_name##_tsd_wrapper_t *wrapper; \ 290 \ 291 assert(a_name##_booted); \ 292 wrapper = a_name##_tsd_get_wrapper(); \ 293 wrapper->val = *(val); \ 294 if (a_cleanup != malloc_tsd_no_cleanup) \ 295 wrapper->initialized = true; \ 296 } 297 #else 298 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 299 a_cleanup) \ 300 /* Data structure. */ \ 301 typedef struct { \ 302 bool initialized; \ 303 a_type val; \ 304 } a_name##_tsd_wrapper_t; \ 305 /* Initialization/cleanup. */ \ 306 a_attr void \ 307 a_name##_tsd_cleanup_wrapper(void *arg) \ 308 { \ 309 a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\ 310 \ 311 if (a_cleanup != malloc_tsd_no_cleanup && \ 312 wrapper->initialized) { \ 313 wrapper->initialized = false; \ 314 a_cleanup(&wrapper->val); \ 315 if (wrapper->initialized) { \ 316 /* Trigger another cleanup round. */ \ 317 if (pthread_setspecific(a_name##_tsd, \ 318 (void *)wrapper)) { \ 319 malloc_write("<jemalloc>: Error" \ 320 " setting TSD for "#a_name"\n"); \ 321 if (opt_abort) \ 322 abort(); \ 323 } \ 324 return; \ 325 } \ 326 } \ 327 malloc_tsd_dalloc(wrapper); \ 328 } \ 329 a_attr bool \ 330 a_name##_tsd_boot(void) \ 331 { \ 332 \ 333 if (pthread_key_create(&a_name##_tsd, \ 334 a_name##_tsd_cleanup_wrapper) != 0) \ 335 return (true); \ 336 a_name##_booted = true; \ 337 return (false); \ 338 } \ 339 /* Get/set. */ \ 340 a_attr a_name##_tsd_wrapper_t * \ 341 a_name##_tsd_get_wrapper(void) \ 342 { \ 343 a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ 344 pthread_getspecific(a_name##_tsd); \ 345 \ 346 if (wrapper == NULL) { \ 347 tsd_init_block_t block; \ 348 wrapper = tsd_init_check_recursion( \ 349 &a_name##_tsd_init_head, &block); \ 350 if (wrapper) \ 351 return (wrapper); \ 352 wrapper = (a_name##_tsd_wrapper_t *) \ 353 malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ 354 block.data = wrapper; \ 355 if (wrapper == NULL) { \ 356 malloc_write("<jemalloc>: Error allocating" \ 357 " TSD for "#a_name"\n"); \ 358 abort(); \ 359 } else { \ 360 static a_type tsd_static_data = a_initializer; \ 361 wrapper->initialized = false; \ 362 wrapper->val = tsd_static_data; \ 363 } \ 364 if (pthread_setspecific(a_name##_tsd, \ 365 (void *)wrapper)) { \ 366 malloc_write("<jemalloc>: Error setting" \ 367 " TSD for "#a_name"\n"); \ 368 abort(); \ 369 } \ 370 tsd_init_finish(&a_name##_tsd_init_head, &block); \ 371 } \ 372 return (wrapper); \ 373 } \ 374 a_attr a_type * \ 375 a_name##_tsd_get(void) \ 376 { \ 377 a_name##_tsd_wrapper_t *wrapper; \ 378 \ 379 assert(a_name##_booted); \ 380 wrapper = a_name##_tsd_get_wrapper(); \ 381 return (&wrapper->val); \ 382 } \ 383 a_attr void \ 384 a_name##_tsd_set(a_type *val) \ 385 { \ 386 a_name##_tsd_wrapper_t *wrapper; \ 387 \ 388 assert(a_name##_booted); \ 389 wrapper = a_name##_tsd_get_wrapper(); \ 390 wrapper->val = *(val); \ 391 if (a_cleanup != malloc_tsd_no_cleanup) \ 392 wrapper->initialized = true; \ 393 } 394 #endif 395 396 #endif /* JEMALLOC_H_TYPES */ 397 /******************************************************************************/ 398 #ifdef JEMALLOC_H_STRUCTS 399 400 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 401 !defined(_WIN32)) 402 struct tsd_init_block_s { 403 ql_elm(tsd_init_block_t) link; 404 pthread_t thread; 405 void *data; 406 }; 407 struct tsd_init_head_s { 408 ql_head(tsd_init_block_t) blocks; 409 malloc_mutex_t lock; 410 }; 411 #endif 412 413 #endif /* JEMALLOC_H_STRUCTS */ 414 /******************************************************************************/ 415 #ifdef JEMALLOC_H_EXTERNS 416 417 void *malloc_tsd_malloc(size_t size); 418 void malloc_tsd_dalloc(void *wrapper); 419 void malloc_tsd_no_cleanup(void *); 420 void malloc_tsd_cleanup_register(bool (*f)(void)); 421 void malloc_tsd_boot(void); 422 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 423 !defined(_WIN32)) 424 void *tsd_init_check_recursion(tsd_init_head_t *head, 425 tsd_init_block_t *block); 426 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); 427 #endif 428 429 #endif /* JEMALLOC_H_EXTERNS */ 430 /******************************************************************************/ 431 #ifdef JEMALLOC_H_INLINES 432 433 #endif /* JEMALLOC_H_INLINES */ 434 /******************************************************************************/ 435