1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 typedef struct tcache_bin_info_s tcache_bin_info_t;
5 typedef struct tcache_bin_s tcache_bin_t;
6 typedef struct tcache_s tcache_t;
7 typedef struct tcaches_s tcaches_t;
8
9 /*
10 * tcache pointers close to NULL are used to encode state information that is
11 * used for two purposes: preventing thread caching on a per thread basis and
12 * cleaning up during thread shutdown.
13 */
14 #define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1)
15 #define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2)
16 #define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
17 #define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
18
19 /*
20 * Absolute minimum number of cache slots for each small bin.
21 */
22 #define TCACHE_NSLOTS_SMALL_MIN 20
23
24 /*
25 * Absolute maximum number of cache slots for each small bin in the thread
26 * cache. This is an additional constraint beyond that imposed as: twice the
27 * number of regions per run for this size class.
28 *
29 * This constant must be an even number.
30 */
31 #if defined(ANDROID_TCACHE_NSLOTS_SMALL_MAX)
32 #define TCACHE_NSLOTS_SMALL_MAX ANDROID_TCACHE_NSLOTS_SMALL_MAX
33 #else
34 #define TCACHE_NSLOTS_SMALL_MAX 200
35 #endif
36
37 /* Number of cache slots for large size classes. */
38 #if defined(ANDROID_TCACHE_NSLOTS_LARGE)
39 #define TCACHE_NSLOTS_LARGE ANDROID_TCACHE_NSLOTS_LARGE
40 #else
41 #define TCACHE_NSLOTS_LARGE 20
42 #endif
43
44 /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
45 #if defined(ANDROID_LG_TCACHE_MAXCLASS_DEFAULT)
46 #define LG_TCACHE_MAXCLASS_DEFAULT ANDROID_LG_TCACHE_MAXCLASS_DEFAULT
47 #else
48 #define LG_TCACHE_MAXCLASS_DEFAULT 15
49 #endif
50
51 /*
52 * TCACHE_GC_SWEEP is the approximate number of allocation events between
53 * full GC sweeps. Integer rounding may cause the actual number to be
54 * slightly higher, since GC is performed incrementally.
55 */
56 #define TCACHE_GC_SWEEP 8192
57
58 /* Number of tcache allocation/deallocation events between incremental GCs. */
59 #define TCACHE_GC_INCR \
60 ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
61
62 #endif /* JEMALLOC_H_TYPES */
63 /******************************************************************************/
64 #ifdef JEMALLOC_H_STRUCTS
65
66 typedef enum {
67 tcache_enabled_false = 0, /* Enable cast to/from bool. */
68 tcache_enabled_true = 1,
69 tcache_enabled_default = 2
70 } tcache_enabled_t;
71
72 /*
73 * Read-only information associated with each element of tcache_t's tbins array
74 * is stored separately, mainly to reduce memory usage.
75 */
76 struct tcache_bin_info_s {
77 unsigned ncached_max; /* Upper limit on ncached. */
78 };
79
80 struct tcache_bin_s {
81 tcache_bin_stats_t tstats;
82 int low_water; /* Min # cached since last GC. */
83 unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */
84 unsigned ncached; /* # of cached objects. */
85 /*
86 * To make use of adjacent cacheline prefetch, the items in the avail
87 * stack goes to higher address for newer allocations. avail points
88 * just above the available space, which means that
89 * avail[-ncached, ... -1] are available items and the lowest item will
90 * be allocated first.
91 */
92 void **avail; /* Stack of available objects. */
93 };
94
95 struct tcache_s {
96 ql_elm(tcache_t) link; /* Used for aggregating stats. */
97 uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */
98 ticker_t gc_ticker; /* Drives incremental GC. */
99 szind_t next_gc_bin; /* Next bin to GC. */
100 tcache_bin_t tbins[1]; /* Dynamically sized. */
101 /*
102 * The pointer stacks associated with tbins follow as a contiguous
103 * array. During tcache initialization, the avail pointer in each
104 * element of tbins is initialized to point to the proper offset within
105 * this array.
106 */
107 };
108
109 /* Linkage for list of available (previously used) explicit tcache IDs. */
110 struct tcaches_s {
111 union {
112 tcache_t *tcache;
113 tcaches_t *next;
114 };
115 };
116
117 #endif /* JEMALLOC_H_STRUCTS */
118 /******************************************************************************/
119 #ifdef JEMALLOC_H_EXTERNS
120
121 extern bool opt_tcache;
122 extern ssize_t opt_lg_tcache_max;
123
124 extern tcache_bin_info_t *tcache_bin_info;
125
126 /*
127 * Number of tcache bins. There are NBINS small-object bins, plus 0 or more
128 * large-object bins.
129 */
130 extern unsigned nhbins;
131
132 /* Maximum cached size class. */
133 extern size_t tcache_maxclass;
134
135 /*
136 * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
137 * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
138 * completely disjoint from this data structure. tcaches starts off as a sparse
139 * array, so it has no physical memory footprint until individual pages are
140 * touched. This allows the entire array to be allocated the first time an
141 * explicit tcache is created without a disproportionate impact on memory usage.
142 */
143 extern tcaches_t *tcaches;
144
145 size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
146 void tcache_event_hard(tsd_t *tsd, tcache_t *tcache);
147 void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
148 tcache_bin_t *tbin, szind_t binind, bool *tcache_success);
149 void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
150 szind_t binind, unsigned rem);
151 void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
152 unsigned rem, tcache_t *tcache);
153 void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
154 arena_t *oldarena, arena_t *newarena);
155 tcache_t *tcache_get_hard(tsd_t *tsd);
156 tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena);
157 void tcache_cleanup(tsd_t *tsd);
158 void tcache_enabled_cleanup(tsd_t *tsd);
159 void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
160 bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
161 void tcaches_flush(tsd_t *tsd, unsigned ind);
162 void tcaches_destroy(tsd_t *tsd, unsigned ind);
163 bool tcache_boot(tsdn_t *tsdn);
164 void tcache_prefork(tsdn_t *tsdn);
165 void tcache_postfork_parent(tsdn_t *tsdn);
166 void tcache_postfork_child(tsdn_t *tsdn);
167
168 #endif /* JEMALLOC_H_EXTERNS */
169 /******************************************************************************/
170 #ifdef JEMALLOC_H_INLINES
171
172 #ifndef JEMALLOC_ENABLE_INLINE
173 void tcache_event(tsd_t *tsd, tcache_t *tcache);
174 void tcache_flush(void);
175 bool tcache_enabled_get(void);
176 tcache_t *tcache_get(tsd_t *tsd, bool create);
177 void tcache_enabled_set(bool enabled);
178 void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
179 void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
180 size_t size, szind_t ind, bool zero, bool slow_path);
181 void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
182 size_t size, szind_t ind, bool zero, bool slow_path);
183 void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
184 szind_t binind, bool slow_path);
185 void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
186 size_t size, bool slow_path);
187 tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
188 #endif
189
190 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
191 JEMALLOC_INLINE void
tcache_flush(void)192 tcache_flush(void)
193 {
194 tsd_t *tsd;
195
196 cassert(config_tcache);
197
198 tsd = tsd_fetch();
199 tcache_cleanup(tsd);
200 }
201
202 JEMALLOC_INLINE bool
tcache_enabled_get(void)203 tcache_enabled_get(void)
204 {
205 tsd_t *tsd;
206 tcache_enabled_t tcache_enabled;
207
208 cassert(config_tcache);
209
210 tsd = tsd_fetch();
211 tcache_enabled = tsd_tcache_enabled_get(tsd);
212 if (tcache_enabled == tcache_enabled_default) {
213 tcache_enabled = (tcache_enabled_t)opt_tcache;
214 tsd_tcache_enabled_set(tsd, tcache_enabled);
215 }
216
217 return ((bool)tcache_enabled);
218 }
219
220 JEMALLOC_INLINE void
tcache_enabled_set(bool enabled)221 tcache_enabled_set(bool enabled)
222 {
223 tsd_t *tsd;
224 tcache_enabled_t tcache_enabled;
225
226 cassert(config_tcache);
227
228 tsd = tsd_fetch();
229
230 tcache_enabled = (tcache_enabled_t)enabled;
231 tsd_tcache_enabled_set(tsd, tcache_enabled);
232
233 if (!enabled)
234 tcache_cleanup(tsd);
235 }
236
237 JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(tsd_t * tsd,bool create)238 tcache_get(tsd_t *tsd, bool create)
239 {
240 tcache_t *tcache;
241
242 if (!config_tcache)
243 return (NULL);
244
245 tcache = tsd_tcache_get(tsd);
246 if (!create)
247 return (tcache);
248 if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
249 tcache = tcache_get_hard(tsd);
250 tsd_tcache_set(tsd, tcache);
251 }
252
253 return (tcache);
254 }
255
256 JEMALLOC_ALWAYS_INLINE void
tcache_event(tsd_t * tsd,tcache_t * tcache)257 tcache_event(tsd_t *tsd, tcache_t *tcache)
258 {
259
260 if (TCACHE_GC_INCR == 0)
261 return;
262
263 if (unlikely(ticker_tick(&tcache->gc_ticker)))
264 tcache_event_hard(tsd, tcache);
265 }
266
267 JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_easy(tcache_bin_t * tbin,bool * tcache_success)268 tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success)
269 {
270 void *ret;
271
272 if (unlikely(tbin->ncached == 0)) {
273 tbin->low_water = -1;
274 *tcache_success = false;
275 return (NULL);
276 }
277 /*
278 * tcache_success (instead of ret) should be checked upon the return of
279 * this function. We avoid checking (ret == NULL) because there is
280 * never a null stored on the avail stack (which is unknown to the
281 * compiler), and eagerly checking ret would cause pipeline stall
282 * (waiting for the cacheline).
283 */
284 *tcache_success = true;
285 ret = *(tbin->avail - tbin->ncached);
286 tbin->ncached--;
287
288 if (unlikely((int)tbin->ncached < tbin->low_water))
289 tbin->low_water = tbin->ncached;
290
291 return (ret);
292 }
293
294 JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_small(tsd_t * tsd,arena_t * arena,tcache_t * tcache,size_t size,szind_t binind,bool zero,bool slow_path)295 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
296 szind_t binind, bool zero, bool slow_path)
297 {
298 void *ret;
299 tcache_bin_t *tbin;
300 bool tcache_success;
301 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
302
303 assert(binind < NBINS);
304 tbin = &tcache->tbins[binind];
305 ret = tcache_alloc_easy(tbin, &tcache_success);
306 assert(tcache_success == (ret != NULL));
307 if (unlikely(!tcache_success)) {
308 bool tcache_hard_success;
309 arena = arena_choose(tsd, arena);
310 if (unlikely(arena == NULL))
311 return (NULL);
312
313 ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache,
314 tbin, binind, &tcache_hard_success);
315 if (tcache_hard_success == false)
316 return (NULL);
317 }
318
319 assert(ret);
320 /*
321 * Only compute usize if required. The checks in the following if
322 * statement are all static.
323 */
324 if (config_prof || (slow_path && config_fill) || unlikely(zero)) {
325 usize = index2size(binind);
326 assert(tcache_salloc(tsd_tsdn(tsd), ret) == usize);
327 }
328
329 if (likely(!zero)) {
330 if (slow_path && config_fill) {
331 if (unlikely(opt_junk_alloc)) {
332 arena_alloc_junk_small(ret,
333 &arena_bin_info[binind], false);
334 } else if (unlikely(opt_zero))
335 memset(ret, 0, usize);
336 }
337 } else {
338 if (slow_path && config_fill && unlikely(opt_junk_alloc)) {
339 arena_alloc_junk_small(ret, &arena_bin_info[binind],
340 true);
341 }
342 memset(ret, 0, usize);
343 }
344
345 if (config_stats)
346 tbin->tstats.nrequests++;
347 if (config_prof)
348 tcache->prof_accumbytes += usize;
349 tcache_event(tsd, tcache);
350 return (ret);
351 }
352
353 JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_large(tsd_t * tsd,arena_t * arena,tcache_t * tcache,size_t size,szind_t binind,bool zero,bool slow_path)354 tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
355 szind_t binind, bool zero, bool slow_path)
356 {
357 void *ret;
358 tcache_bin_t *tbin;
359 bool tcache_success;
360
361 assert(binind < nhbins);
362 tbin = &tcache->tbins[binind];
363 ret = tcache_alloc_easy(tbin, &tcache_success);
364 assert(tcache_success == (ret != NULL));
365 if (unlikely(!tcache_success)) {
366 /*
367 * Only allocate one large object at a time, because it's quite
368 * expensive to create one and not use it.
369 */
370 arena = arena_choose(tsd, arena);
371 if (unlikely(arena == NULL))
372 return (NULL);
373
374 ret = arena_malloc_large(tsd_tsdn(tsd), arena, binind, zero);
375 if (ret == NULL)
376 return (NULL);
377 } else {
378 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
379
380 /* Only compute usize on demand */
381 if (config_prof || (slow_path && config_fill) ||
382 unlikely(zero)) {
383 usize = index2size(binind);
384 assert(usize <= tcache_maxclass);
385 }
386
387 if (config_prof && usize == LARGE_MINCLASS) {
388 arena_chunk_t *chunk =
389 (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
390 size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
391 LG_PAGE);
392 arena_mapbits_large_binind_set(chunk, pageind,
393 BININD_INVALID);
394 }
395 if (likely(!zero)) {
396 if (slow_path && config_fill) {
397 if (unlikely(opt_junk_alloc)) {
398 memset(ret, JEMALLOC_ALLOC_JUNK,
399 usize);
400 } else if (unlikely(opt_zero))
401 memset(ret, 0, usize);
402 }
403 } else
404 memset(ret, 0, usize);
405
406 if (config_stats)
407 tbin->tstats.nrequests++;
408 if (config_prof)
409 tcache->prof_accumbytes += usize;
410 }
411
412 tcache_event(tsd, tcache);
413 return (ret);
414 }
415
416 JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_small(tsd_t * tsd,tcache_t * tcache,void * ptr,szind_t binind,bool slow_path)417 tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
418 bool slow_path)
419 {
420 tcache_bin_t *tbin;
421 tcache_bin_info_t *tbin_info;
422
423 assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS);
424
425 if (slow_path && config_fill && unlikely(opt_junk_free))
426 arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
427
428 tbin = &tcache->tbins[binind];
429 tbin_info = &tcache_bin_info[binind];
430 if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
431 tcache_bin_flush_small(tsd, tcache, tbin, binind,
432 (tbin_info->ncached_max >> 1));
433 }
434 assert(tbin->ncached < tbin_info->ncached_max);
435 tbin->ncached++;
436 *(tbin->avail - tbin->ncached) = ptr;
437
438 tcache_event(tsd, tcache);
439 }
440
441 JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_large(tsd_t * tsd,tcache_t * tcache,void * ptr,size_t size,bool slow_path)442 tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
443 bool slow_path)
444 {
445 szind_t binind;
446 tcache_bin_t *tbin;
447 tcache_bin_info_t *tbin_info;
448
449 assert((size & PAGE_MASK) == 0);
450 assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS);
451 assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass);
452
453 binind = size2index(size);
454
455 if (slow_path && config_fill && unlikely(opt_junk_free))
456 arena_dalloc_junk_large(ptr, size);
457
458 tbin = &tcache->tbins[binind];
459 tbin_info = &tcache_bin_info[binind];
460 if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
461 tcache_bin_flush_large(tsd, tbin, binind,
462 (tbin_info->ncached_max >> 1), tcache);
463 }
464 assert(tbin->ncached < tbin_info->ncached_max);
465 tbin->ncached++;
466 *(tbin->avail - tbin->ncached) = ptr;
467
468 tcache_event(tsd, tcache);
469 }
470
471 JEMALLOC_ALWAYS_INLINE tcache_t *
tcaches_get(tsd_t * tsd,unsigned ind)472 tcaches_get(tsd_t *tsd, unsigned ind)
473 {
474 tcaches_t *elm = &tcaches[ind];
475 if (unlikely(elm->tcache == NULL)) {
476 elm->tcache = tcache_create(tsd_tsdn(tsd), arena_choose(tsd,
477 NULL));
478 }
479 return (elm->tcache);
480 }
481 #endif
482
483 #endif /* JEMALLOC_H_INLINES */
484 /******************************************************************************/
485