1 #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
2 #define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
3
4 #include "jemalloc/internal/jemalloc_internal_types.h"
5 #include "jemalloc/internal/mutex.h"
6 #include "jemalloc/internal/rtree.h"
7 #include "jemalloc/internal/size_classes.h"
8 #include "jemalloc/internal/sz.h"
9 #include "jemalloc/internal/ticker.h"
10
11 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t * tsdn,const void * ptr,alloc_ctx_t * alloc_ctx)12 arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
13 cassert(config_prof);
14 assert(ptr != NULL);
15
16 /* Static check. */
17 if (alloc_ctx == NULL) {
18 const extent_t *extent = iealloc(tsdn, ptr);
19 if (unlikely(!extent_slab_get(extent))) {
20 return large_prof_tctx_get(tsdn, extent);
21 }
22 } else {
23 if (unlikely(!alloc_ctx->slab)) {
24 return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr));
25 }
26 }
27 return (prof_tctx_t *)(uintptr_t)1U;
28 }
29
30 JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_set(tsdn_t * tsdn,const void * ptr,UNUSED size_t usize,alloc_ctx_t * alloc_ctx,prof_tctx_t * tctx)31 arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize,
32 alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
33 cassert(config_prof);
34 assert(ptr != NULL);
35
36 /* Static check. */
37 if (alloc_ctx == NULL) {
38 extent_t *extent = iealloc(tsdn, ptr);
39 if (unlikely(!extent_slab_get(extent))) {
40 large_prof_tctx_set(tsdn, extent, tctx);
41 }
42 } else {
43 if (unlikely(!alloc_ctx->slab)) {
44 large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx);
45 }
46 }
47 }
48
49 static inline void
arena_prof_tctx_reset(tsdn_t * tsdn,const void * ptr,UNUSED prof_tctx_t * tctx)50 arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) {
51 cassert(config_prof);
52 assert(ptr != NULL);
53
54 extent_t *extent = iealloc(tsdn, ptr);
55 assert(!extent_slab_get(extent));
56
57 large_prof_tctx_reset(tsdn, extent);
58 }
59
60 JEMALLOC_ALWAYS_INLINE void
arena_decay_ticks(tsdn_t * tsdn,arena_t * arena,unsigned nticks)61 arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
62 tsd_t *tsd;
63 ticker_t *decay_ticker;
64
65 if (unlikely(tsdn_null(tsdn))) {
66 return;
67 }
68 tsd = tsdn_tsd(tsdn);
69 decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena));
70 if (unlikely(decay_ticker == NULL)) {
71 return;
72 }
73 if (unlikely(ticker_ticks(decay_ticker, nticks))) {
74 arena_decay(tsdn, arena, false, false);
75 }
76 }
77
78 JEMALLOC_ALWAYS_INLINE void
arena_decay_tick(tsdn_t * tsdn,arena_t * arena)79 arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
80 malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx);
81 malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx);
82
83 arena_decay_ticks(tsdn, arena, 1);
84 }
85
86 JEMALLOC_ALWAYS_INLINE void *
arena_malloc(tsdn_t * tsdn,arena_t * arena,size_t size,szind_t ind,bool zero,tcache_t * tcache,bool slow_path)87 arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
88 tcache_t *tcache, bool slow_path) {
89 assert(!tsdn_null(tsdn) || tcache == NULL);
90 assert(size != 0);
91
92 if (likely(tcache != NULL)) {
93 if (likely(size <= SMALL_MAXCLASS)) {
94 return tcache_alloc_small(tsdn_tsd(tsdn), arena,
95 tcache, size, ind, zero, slow_path);
96 }
97 if (likely(size <= tcache_maxclass)) {
98 return tcache_alloc_large(tsdn_tsd(tsdn), arena,
99 tcache, size, ind, zero, slow_path);
100 }
101 /* (size > tcache_maxclass) case falls through. */
102 assert(size > tcache_maxclass);
103 }
104
105 return arena_malloc_hard(tsdn, arena, size, ind, zero);
106 }
107
108 JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t * tsdn,const void * ptr)109 arena_aalloc(tsdn_t *tsdn, const void *ptr) {
110 return extent_arena_get(iealloc(tsdn, ptr));
111 }
112
113 JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t * tsdn,const void * ptr)114 arena_salloc(tsdn_t *tsdn, const void *ptr) {
115 assert(ptr != NULL);
116
117 rtree_ctx_t rtree_ctx_fallback;
118 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
119
120 szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx,
121 (uintptr_t)ptr, true);
122 assert(szind != NSIZES);
123
124 return sz_index2size(szind);
125 }
126
127 JEMALLOC_ALWAYS_INLINE size_t
arena_vsalloc(tsdn_t * tsdn,const void * ptr)128 arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
129 /*
130 * Return 0 if ptr is not within an extent managed by jemalloc. This
131 * function has two extra costs relative to isalloc():
132 * - The rtree calls cannot claim to be dependent lookups, which induces
133 * rtree lookup load dependencies.
134 * - The lookup may fail, so there is an extra branch to check for
135 * failure.
136 */
137
138 rtree_ctx_t rtree_ctx_fallback;
139 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
140
141 extent_t *extent;
142 szind_t szind;
143 if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx,
144 (uintptr_t)ptr, false, &extent, &szind)) {
145 return 0;
146 }
147
148 if (extent == NULL) {
149 return 0;
150 }
151 assert(extent_state_get(extent) == extent_state_active);
152 /* Only slab members should be looked up via interior pointers. */
153 assert(extent_addr_get(extent) == ptr || extent_slab_get(extent));
154
155 assert(szind != NSIZES);
156
157 return sz_index2size(szind);
158 }
159
160 static inline void
arena_dalloc_no_tcache(tsdn_t * tsdn,void * ptr)161 arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
162 assert(ptr != NULL);
163
164 rtree_ctx_t rtree_ctx_fallback;
165 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
166
167 szind_t szind;
168 bool slab;
169 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
170 true, &szind, &slab);
171
172 if (config_debug) {
173 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
174 rtree_ctx, (uintptr_t)ptr, true);
175 assert(szind == extent_szind_get(extent));
176 assert(szind < NSIZES);
177 assert(slab == extent_slab_get(extent));
178 }
179
180 if (likely(slab)) {
181 /* Small allocation. */
182 arena_dalloc_small(tsdn, ptr);
183 } else {
184 extent_t *extent = iealloc(tsdn, ptr);
185 large_dalloc(tsdn, extent);
186 }
187 }
188
189 JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t * tsdn,void * ptr,tcache_t * tcache,alloc_ctx_t * alloc_ctx,bool slow_path)190 arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
191 alloc_ctx_t *alloc_ctx, bool slow_path) {
192 assert(!tsdn_null(tsdn) || tcache == NULL);
193 assert(ptr != NULL);
194
195 if (unlikely(tcache == NULL)) {
196 arena_dalloc_no_tcache(tsdn, ptr);
197 return;
198 }
199
200 szind_t szind;
201 bool slab;
202 rtree_ctx_t *rtree_ctx;
203 if (alloc_ctx != NULL) {
204 szind = alloc_ctx->szind;
205 slab = alloc_ctx->slab;
206 assert(szind != NSIZES);
207 } else {
208 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
209 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
210 (uintptr_t)ptr, true, &szind, &slab);
211 }
212
213 if (config_debug) {
214 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
215 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
216 rtree_ctx, (uintptr_t)ptr, true);
217 assert(szind == extent_szind_get(extent));
218 assert(szind < NSIZES);
219 assert(slab == extent_slab_get(extent));
220 }
221
222 if (likely(slab)) {
223 /* Small allocation. */
224 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
225 slow_path);
226 } else {
227 if (szind < nhbins) {
228 if (config_prof && unlikely(szind < NBINS)) {
229 arena_dalloc_promoted(tsdn, ptr, tcache,
230 slow_path);
231 } else {
232 tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
233 szind, slow_path);
234 }
235 } else {
236 extent_t *extent = iealloc(tsdn, ptr);
237 large_dalloc(tsdn, extent);
238 }
239 }
240 }
241
242 static inline void
arena_sdalloc_no_tcache(tsdn_t * tsdn,void * ptr,size_t size)243 arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
244 assert(ptr != NULL);
245 assert(size <= LARGE_MAXCLASS);
246
247 szind_t szind;
248 bool slab;
249 if (!config_prof || !opt_prof) {
250 /*
251 * There is no risk of being confused by a promoted sampled
252 * object, so base szind and slab on the given size.
253 */
254 szind = sz_size2index(size);
255 slab = (szind < NBINS);
256 }
257
258 if ((config_prof && opt_prof) || config_debug) {
259 rtree_ctx_t rtree_ctx_fallback;
260 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
261 &rtree_ctx_fallback);
262
263 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
264 (uintptr_t)ptr, true, &szind, &slab);
265
266 assert(szind == sz_size2index(size));
267 assert((config_prof && opt_prof) || slab == (szind < NBINS));
268
269 if (config_debug) {
270 extent_t *extent = rtree_extent_read(tsdn,
271 &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
272 assert(szind == extent_szind_get(extent));
273 assert(slab == extent_slab_get(extent));
274 }
275 }
276
277 if (likely(slab)) {
278 /* Small allocation. */
279 arena_dalloc_small(tsdn, ptr);
280 } else {
281 extent_t *extent = iealloc(tsdn, ptr);
282 large_dalloc(tsdn, extent);
283 }
284 }
285
286 JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t * tsdn,void * ptr,size_t size,tcache_t * tcache,alloc_ctx_t * alloc_ctx,bool slow_path)287 arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
288 alloc_ctx_t *alloc_ctx, bool slow_path) {
289 assert(!tsdn_null(tsdn) || tcache == NULL);
290 assert(ptr != NULL);
291 assert(size <= LARGE_MAXCLASS);
292
293 if (unlikely(tcache == NULL)) {
294 arena_sdalloc_no_tcache(tsdn, ptr, size);
295 return;
296 }
297
298 szind_t szind;
299 bool slab;
300 UNUSED alloc_ctx_t local_ctx;
301 if (config_prof && opt_prof) {
302 if (alloc_ctx == NULL) {
303 /* Uncommon case and should be a static check. */
304 rtree_ctx_t rtree_ctx_fallback;
305 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
306 &rtree_ctx_fallback);
307 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
308 (uintptr_t)ptr, true, &local_ctx.szind,
309 &local_ctx.slab);
310 assert(local_ctx.szind == sz_size2index(size));
311 alloc_ctx = &local_ctx;
312 }
313 slab = alloc_ctx->slab;
314 szind = alloc_ctx->szind;
315 } else {
316 /*
317 * There is no risk of being confused by a promoted sampled
318 * object, so base szind and slab on the given size.
319 */
320 szind = sz_size2index(size);
321 slab = (szind < NBINS);
322 }
323
324 if (config_debug) {
325 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
326 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
327 (uintptr_t)ptr, true, &szind, &slab);
328 extent_t *extent = rtree_extent_read(tsdn,
329 &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
330 assert(szind == extent_szind_get(extent));
331 assert(slab == extent_slab_get(extent));
332 }
333
334 if (likely(slab)) {
335 /* Small allocation. */
336 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
337 slow_path);
338 } else {
339 if (szind < nhbins) {
340 if (config_prof && unlikely(szind < NBINS)) {
341 arena_dalloc_promoted(tsdn, ptr, tcache,
342 slow_path);
343 } else {
344 tcache_dalloc_large(tsdn_tsd(tsdn),
345 tcache, ptr, szind, slow_path);
346 }
347 } else {
348 extent_t *extent = iealloc(tsdn, ptr);
349 large_dalloc(tsdn, extent);
350 }
351 }
352 }
353
354 #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */
355