1 #ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H
2 #define JEMALLOC_INTERNAL_EXTENT_INLINES_H
3
4 #include "jemalloc/internal/mutex.h"
5 #include "jemalloc/internal/mutex_pool.h"
6 #include "jemalloc/internal/pages.h"
7 #include "jemalloc/internal/prng.h"
8 #include "jemalloc/internal/ql.h"
9 #include "jemalloc/internal/sz.h"
10
11 static inline void
extent_lock(tsdn_t * tsdn,extent_t * extent)12 extent_lock(tsdn_t *tsdn, extent_t *extent) {
13 assert(extent != NULL);
14 mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
15 }
16
17 static inline void
extent_unlock(tsdn_t * tsdn,extent_t * extent)18 extent_unlock(tsdn_t *tsdn, extent_t *extent) {
19 assert(extent != NULL);
20 mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
21 }
22
23 static inline void
extent_lock2(tsdn_t * tsdn,extent_t * extent1,extent_t * extent2)24 extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
25 assert(extent1 != NULL && extent2 != NULL);
26 mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
27 (uintptr_t)extent2);
28 }
29
30 static inline void
extent_unlock2(tsdn_t * tsdn,extent_t * extent1,extent_t * extent2)31 extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
32 assert(extent1 != NULL && extent2 != NULL);
33 mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
34 (uintptr_t)extent2);
35 }
36
37 static inline arena_t *
extent_arena_get(const extent_t * extent)38 extent_arena_get(const extent_t *extent) {
39 unsigned arena_ind = (unsigned)((extent->e_bits &
40 EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT);
41 /*
42 * The following check is omitted because we should never actually read
43 * a NULL arena pointer.
44 */
45 if (false && arena_ind >= MALLOCX_ARENA_LIMIT) {
46 return NULL;
47 }
48 assert(arena_ind < MALLOCX_ARENA_LIMIT);
49 return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE);
50 }
51
52 static inline szind_t
extent_szind_get_maybe_invalid(const extent_t * extent)53 extent_szind_get_maybe_invalid(const extent_t *extent) {
54 szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >>
55 EXTENT_BITS_SZIND_SHIFT);
56 assert(szind <= NSIZES);
57 return szind;
58 }
59
60 static inline szind_t
extent_szind_get(const extent_t * extent)61 extent_szind_get(const extent_t *extent) {
62 szind_t szind = extent_szind_get_maybe_invalid(extent);
63 assert(szind < NSIZES); /* Never call when "invalid". */
64 return szind;
65 }
66
67 static inline size_t
extent_usize_get(const extent_t * extent)68 extent_usize_get(const extent_t *extent) {
69 return sz_index2size(extent_szind_get(extent));
70 }
71
72 static inline size_t
extent_sn_get(const extent_t * extent)73 extent_sn_get(const extent_t *extent) {
74 return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >>
75 EXTENT_BITS_SN_SHIFT);
76 }
77
78 static inline extent_state_t
extent_state_get(const extent_t * extent)79 extent_state_get(const extent_t *extent) {
80 return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK) >>
81 EXTENT_BITS_STATE_SHIFT);
82 }
83
84 static inline bool
extent_zeroed_get(const extent_t * extent)85 extent_zeroed_get(const extent_t *extent) {
86 return (bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK) >>
87 EXTENT_BITS_ZEROED_SHIFT);
88 }
89
90 static inline bool
extent_committed_get(const extent_t * extent)91 extent_committed_get(const extent_t *extent) {
92 return (bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK) >>
93 EXTENT_BITS_COMMITTED_SHIFT);
94 }
95
96 static inline bool
extent_dumpable_get(const extent_t * extent)97 extent_dumpable_get(const extent_t *extent) {
98 return (bool)((extent->e_bits & EXTENT_BITS_DUMPABLE_MASK) >>
99 EXTENT_BITS_DUMPABLE_SHIFT);
100 }
101
102 static inline bool
extent_slab_get(const extent_t * extent)103 extent_slab_get(const extent_t *extent) {
104 return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >>
105 EXTENT_BITS_SLAB_SHIFT);
106 }
107
108 static inline unsigned
extent_nfree_get(const extent_t * extent)109 extent_nfree_get(const extent_t *extent) {
110 assert(extent_slab_get(extent));
111 return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >>
112 EXTENT_BITS_NFREE_SHIFT);
113 }
114
115 static inline void *
extent_base_get(const extent_t * extent)116 extent_base_get(const extent_t *extent) {
117 assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
118 !extent_slab_get(extent));
119 return PAGE_ADDR2BASE(extent->e_addr);
120 }
121
122 static inline void *
extent_addr_get(const extent_t * extent)123 extent_addr_get(const extent_t *extent) {
124 assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
125 !extent_slab_get(extent));
126 return extent->e_addr;
127 }
128
129 static inline size_t
extent_size_get(const extent_t * extent)130 extent_size_get(const extent_t *extent) {
131 return (extent->e_size_esn & EXTENT_SIZE_MASK);
132 }
133
134 static inline size_t
extent_esn_get(const extent_t * extent)135 extent_esn_get(const extent_t *extent) {
136 return (extent->e_size_esn & EXTENT_ESN_MASK);
137 }
138
139 static inline size_t
extent_bsize_get(const extent_t * extent)140 extent_bsize_get(const extent_t *extent) {
141 return extent->e_bsize;
142 }
143
144 static inline void *
extent_before_get(const extent_t * extent)145 extent_before_get(const extent_t *extent) {
146 return (void *)((uintptr_t)extent_base_get(extent) - PAGE);
147 }
148
149 static inline void *
extent_last_get(const extent_t * extent)150 extent_last_get(const extent_t *extent) {
151 return (void *)((uintptr_t)extent_base_get(extent) +
152 extent_size_get(extent) - PAGE);
153 }
154
155 static inline void *
extent_past_get(const extent_t * extent)156 extent_past_get(const extent_t *extent) {
157 return (void *)((uintptr_t)extent_base_get(extent) +
158 extent_size_get(extent));
159 }
160
161 static inline arena_slab_data_t *
extent_slab_data_get(extent_t * extent)162 extent_slab_data_get(extent_t *extent) {
163 assert(extent_slab_get(extent));
164 return &extent->e_slab_data;
165 }
166
167 static inline const arena_slab_data_t *
extent_slab_data_get_const(const extent_t * extent)168 extent_slab_data_get_const(const extent_t *extent) {
169 assert(extent_slab_get(extent));
170 return &extent->e_slab_data;
171 }
172
173 static inline prof_tctx_t *
extent_prof_tctx_get(const extent_t * extent)174 extent_prof_tctx_get(const extent_t *extent) {
175 return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx,
176 ATOMIC_ACQUIRE);
177 }
178
179 static inline void
extent_arena_set(extent_t * extent,arena_t * arena)180 extent_arena_set(extent_t *extent, arena_t *arena) {
181 unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U <<
182 MALLOCX_ARENA_BITS) - 1);
183 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK) |
184 ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT);
185 }
186
187 static inline void
extent_addr_set(extent_t * extent,void * addr)188 extent_addr_set(extent_t *extent, void *addr) {
189 extent->e_addr = addr;
190 }
191
192 static inline void
extent_addr_randomize(UNUSED tsdn_t * tsdn,extent_t * extent,size_t alignment)193 extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) {
194 assert(extent_base_get(extent) == extent_addr_get(extent));
195
196 if (alignment < PAGE) {
197 unsigned lg_range = LG_PAGE -
198 lg_floor(CACHELINE_CEILING(alignment));
199 size_t r;
200 if (!tsdn_null(tsdn)) {
201 tsd_t *tsd = tsdn_tsd(tsdn);
202 r = (size_t)prng_lg_range_u64(
203 tsd_offset_statep_get(tsd), lg_range);
204 } else {
205 r = prng_lg_range_zu(
206 &extent_arena_get(extent)->offset_state,
207 lg_range, true);
208 }
209 uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
210 lg_range);
211 extent->e_addr = (void *)((uintptr_t)extent->e_addr +
212 random_offset);
213 assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
214 extent->e_addr);
215 }
216 }
217
218 static inline void
extent_size_set(extent_t * extent,size_t size)219 extent_size_set(extent_t *extent, size_t size) {
220 assert((size & ~EXTENT_SIZE_MASK) == 0);
221 extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK);
222 }
223
224 static inline void
extent_esn_set(extent_t * extent,size_t esn)225 extent_esn_set(extent_t *extent, size_t esn) {
226 extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK) | (esn &
227 EXTENT_ESN_MASK);
228 }
229
230 static inline void
extent_bsize_set(extent_t * extent,size_t bsize)231 extent_bsize_set(extent_t *extent, size_t bsize) {
232 extent->e_bsize = bsize;
233 }
234
235 static inline void
extent_szind_set(extent_t * extent,szind_t szind)236 extent_szind_set(extent_t *extent, szind_t szind) {
237 assert(szind <= NSIZES); /* NSIZES means "invalid". */
238 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) |
239 ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
240 }
241
242 static inline void
extent_nfree_set(extent_t * extent,unsigned nfree)243 extent_nfree_set(extent_t *extent, unsigned nfree) {
244 assert(extent_slab_get(extent));
245 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) |
246 ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
247 }
248
249 static inline void
extent_nfree_inc(extent_t * extent)250 extent_nfree_inc(extent_t *extent) {
251 assert(extent_slab_get(extent));
252 extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
253 }
254
255 static inline void
extent_nfree_dec(extent_t * extent)256 extent_nfree_dec(extent_t *extent) {
257 assert(extent_slab_get(extent));
258 extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
259 }
260
261 static inline void
extent_sn_set(extent_t * extent,size_t sn)262 extent_sn_set(extent_t *extent, size_t sn) {
263 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
264 ((uint64_t)sn << EXTENT_BITS_SN_SHIFT);
265 }
266
267 static inline void
extent_state_set(extent_t * extent,extent_state_t state)268 extent_state_set(extent_t *extent, extent_state_t state) {
269 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) |
270 ((uint64_t)state << EXTENT_BITS_STATE_SHIFT);
271 }
272
273 static inline void
extent_zeroed_set(extent_t * extent,bool zeroed)274 extent_zeroed_set(extent_t *extent, bool zeroed) {
275 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK) |
276 ((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT);
277 }
278
279 static inline void
extent_committed_set(extent_t * extent,bool committed)280 extent_committed_set(extent_t *extent, bool committed) {
281 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK) |
282 ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT);
283 }
284
285 static inline void
extent_dumpable_set(extent_t * extent,bool dumpable)286 extent_dumpable_set(extent_t *extent, bool dumpable) {
287 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_DUMPABLE_MASK) |
288 ((uint64_t)dumpable << EXTENT_BITS_DUMPABLE_SHIFT);
289 }
290
291 static inline void
extent_slab_set(extent_t * extent,bool slab)292 extent_slab_set(extent_t *extent, bool slab) {
293 extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) |
294 ((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT);
295 }
296
297 static inline void
extent_prof_tctx_set(extent_t * extent,prof_tctx_t * tctx)298 extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) {
299 atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE);
300 }
301
302 static inline void
extent_init(extent_t * extent,arena_t * arena,void * addr,size_t size,bool slab,szind_t szind,size_t sn,extent_state_t state,bool zeroed,bool committed,bool dumpable)303 extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
304 bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
305 bool committed, bool dumpable) {
306 assert(addr == PAGE_ADDR2BASE(addr) || !slab);
307
308 extent_arena_set(extent, arena);
309 extent_addr_set(extent, addr);
310 extent_size_set(extent, size);
311 extent_slab_set(extent, slab);
312 extent_szind_set(extent, szind);
313 extent_sn_set(extent, sn);
314 extent_state_set(extent, state);
315 extent_zeroed_set(extent, zeroed);
316 extent_committed_set(extent, committed);
317 extent_dumpable_set(extent, dumpable);
318 ql_elm_new(extent, ql_link);
319 if (config_prof) {
320 extent_prof_tctx_set(extent, NULL);
321 }
322 }
323
324 static inline void
extent_binit(extent_t * extent,void * addr,size_t bsize,size_t sn)325 extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) {
326 extent_arena_set(extent, NULL);
327 extent_addr_set(extent, addr);
328 extent_bsize_set(extent, bsize);
329 extent_slab_set(extent, false);
330 extent_szind_set(extent, NSIZES);
331 extent_sn_set(extent, sn);
332 extent_state_set(extent, extent_state_active);
333 extent_zeroed_set(extent, true);
334 extent_committed_set(extent, true);
335 extent_dumpable_set(extent, true);
336 }
337
338 static inline void
extent_list_init(extent_list_t * list)339 extent_list_init(extent_list_t *list) {
340 ql_new(list);
341 }
342
343 static inline extent_t *
extent_list_first(const extent_list_t * list)344 extent_list_first(const extent_list_t *list) {
345 return ql_first(list);
346 }
347
348 static inline extent_t *
extent_list_last(const extent_list_t * list)349 extent_list_last(const extent_list_t *list) {
350 return ql_last(list, ql_link);
351 }
352
353 static inline void
extent_list_append(extent_list_t * list,extent_t * extent)354 extent_list_append(extent_list_t *list, extent_t *extent) {
355 ql_tail_insert(list, extent, ql_link);
356 }
357
358 static inline void
extent_list_prepend(extent_list_t * list,extent_t * extent)359 extent_list_prepend(extent_list_t *list, extent_t *extent) {
360 ql_head_insert(list, extent, ql_link);
361 }
362
363 static inline void
extent_list_replace(extent_list_t * list,extent_t * to_remove,extent_t * to_insert)364 extent_list_replace(extent_list_t *list, extent_t *to_remove,
365 extent_t *to_insert) {
366 ql_after_insert(to_remove, to_insert, ql_link);
367 ql_remove(list, to_remove, ql_link);
368 }
369
370 static inline void
extent_list_remove(extent_list_t * list,extent_t * extent)371 extent_list_remove(extent_list_t *list, extent_t *extent) {
372 ql_remove(list, extent, ql_link);
373 }
374
375 static inline int
extent_sn_comp(const extent_t * a,const extent_t * b)376 extent_sn_comp(const extent_t *a, const extent_t *b) {
377 size_t a_sn = extent_sn_get(a);
378 size_t b_sn = extent_sn_get(b);
379
380 return (a_sn > b_sn) - (a_sn < b_sn);
381 }
382
383 static inline int
extent_esn_comp(const extent_t * a,const extent_t * b)384 extent_esn_comp(const extent_t *a, const extent_t *b) {
385 size_t a_esn = extent_esn_get(a);
386 size_t b_esn = extent_esn_get(b);
387
388 return (a_esn > b_esn) - (a_esn < b_esn);
389 }
390
391 static inline int
extent_ad_comp(const extent_t * a,const extent_t * b)392 extent_ad_comp(const extent_t *a, const extent_t *b) {
393 uintptr_t a_addr = (uintptr_t)extent_addr_get(a);
394 uintptr_t b_addr = (uintptr_t)extent_addr_get(b);
395
396 return (a_addr > b_addr) - (a_addr < b_addr);
397 }
398
399 static inline int
extent_ead_comp(const extent_t * a,const extent_t * b)400 extent_ead_comp(const extent_t *a, const extent_t *b) {
401 uintptr_t a_eaddr = (uintptr_t)a;
402 uintptr_t b_eaddr = (uintptr_t)b;
403
404 return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr);
405 }
406
407 static inline int
extent_snad_comp(const extent_t * a,const extent_t * b)408 extent_snad_comp(const extent_t *a, const extent_t *b) {
409 int ret;
410
411 ret = extent_sn_comp(a, b);
412 if (ret != 0) {
413 return ret;
414 }
415
416 ret = extent_ad_comp(a, b);
417 return ret;
418 }
419
420 static inline int
extent_esnead_comp(const extent_t * a,const extent_t * b)421 extent_esnead_comp(const extent_t *a, const extent_t *b) {
422 int ret;
423
424 ret = extent_esn_comp(a, b);
425 if (ret != 0) {
426 return ret;
427 }
428
429 ret = extent_ead_comp(a, b);
430 return ret;
431 }
432
433 #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */
434