1 #define JEMALLOC_HUGE_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /******************************************************************************/
5 /* Data. */
6
7 /* Protects chunk-related data structures. */
8 static malloc_mutex_t huge_mtx;
9
10 /******************************************************************************/
11
12 /* Tree of chunks that are stand-alone huge allocations. */
13 static extent_tree_t huge;
14
15 void *
huge_malloc(arena_t * arena,size_t size,bool zero)16 huge_malloc(arena_t *arena, size_t size, bool zero)
17 {
18
19 return (huge_palloc(arena, size, chunksize, zero));
20 }
21
22 void *
huge_palloc(arena_t * arena,size_t size,size_t alignment,bool zero)23 huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
24 {
25 void *ret;
26 size_t csize;
27 extent_node_t *node;
28 bool is_zeroed;
29
30 /* Allocate one or more contiguous chunks for this request. */
31
32 csize = CHUNK_CEILING(size);
33 if (csize == 0) {
34 /* size is large enough to cause size_t wrap-around. */
35 return (NULL);
36 }
37
38 /* Allocate an extent node with which to track the chunk. */
39 node = base_node_alloc();
40 if (node == NULL)
41 return (NULL);
42
43 /*
44 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
45 * it is possible to make correct junk/zero fill decisions below.
46 */
47 is_zeroed = zero;
48 arena = choose_arena(arena);
49 ret = arena_chunk_alloc_huge(arena, csize, alignment, &is_zeroed);
50 if (ret == NULL) {
51 base_node_dalloc(node);
52 return (NULL);
53 }
54
55 /* Insert node into huge. */
56 node->addr = ret;
57 node->size = csize;
58 node->arena = arena;
59
60 malloc_mutex_lock(&huge_mtx);
61 extent_tree_ad_insert(&huge, node);
62 malloc_mutex_unlock(&huge_mtx);
63
64 if (config_fill && zero == false) {
65 if (opt_junk)
66 memset(ret, 0xa5, csize);
67 else if (opt_zero && is_zeroed == false)
68 memset(ret, 0, csize);
69 }
70
71 return (ret);
72 }
73
74 bool
huge_ralloc_no_move(void * ptr,size_t oldsize,size_t size,size_t extra)75 huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
76 {
77
78 /*
79 * Avoid moving the allocation if the size class can be left the same.
80 */
81 if (oldsize > arena_maxclass
82 && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
83 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
84 assert(CHUNK_CEILING(oldsize) == oldsize);
85 return (false);
86 }
87
88 /* Reallocation would require a move. */
89 return (true);
90 }
91
92 void *
huge_ralloc(arena_t * arena,void * ptr,size_t oldsize,size_t size,size_t extra,size_t alignment,bool zero,bool try_tcache_dalloc)93 huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
94 size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc)
95 {
96 void *ret;
97 size_t copysize;
98
99 /* Try to avoid moving the allocation. */
100 if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false)
101 return (ptr);
102
103 /*
104 * size and oldsize are different enough that we need to use a
105 * different size class. In that case, fall back to allocating new
106 * space and copying.
107 */
108 if (alignment > chunksize)
109 ret = huge_palloc(arena, size + extra, alignment, zero);
110 else
111 ret = huge_malloc(arena, size + extra, zero);
112
113 if (ret == NULL) {
114 if (extra == 0)
115 return (NULL);
116 /* Try again, this time without extra. */
117 if (alignment > chunksize)
118 ret = huge_palloc(arena, size, alignment, zero);
119 else
120 ret = huge_malloc(arena, size, zero);
121
122 if (ret == NULL)
123 return (NULL);
124 }
125
126 /*
127 * Copy at most size bytes (not size+extra), since the caller has no
128 * expectation that the extra bytes will be reliably preserved.
129 */
130 copysize = (size < oldsize) ? size : oldsize;
131 memcpy(ret, ptr, copysize);
132 iqalloct(ptr, try_tcache_dalloc);
133 return (ret);
134 }
135
136 #ifdef JEMALLOC_JET
137 #undef huge_dalloc_junk
138 #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
139 #endif
140 static void
huge_dalloc_junk(void * ptr,size_t usize)141 huge_dalloc_junk(void *ptr, size_t usize)
142 {
143
144 if (config_fill && have_dss && opt_junk) {
145 /*
146 * Only bother junk filling if the chunk isn't about to be
147 * unmapped.
148 */
149 if (config_munmap == false || (have_dss && chunk_in_dss(ptr)))
150 memset(ptr, 0x5a, usize);
151 }
152 }
153 #ifdef JEMALLOC_JET
154 #undef huge_dalloc_junk
155 #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
156 huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
157 #endif
158
159 void
huge_dalloc(void * ptr)160 huge_dalloc(void *ptr)
161 {
162 extent_node_t *node, key;
163
164 malloc_mutex_lock(&huge_mtx);
165
166 /* Extract from tree of huge allocations. */
167 key.addr = ptr;
168 node = extent_tree_ad_search(&huge, &key);
169 assert(node != NULL);
170 assert(node->addr == ptr);
171 extent_tree_ad_remove(&huge, node);
172
173 malloc_mutex_unlock(&huge_mtx);
174
175 huge_dalloc_junk(node->addr, node->size);
176 arena_chunk_dalloc_huge(node->arena, node->addr, node->size);
177 base_node_dalloc(node);
178 }
179
180 size_t
huge_salloc(const void * ptr)181 huge_salloc(const void *ptr)
182 {
183 size_t ret;
184 extent_node_t *node, key;
185
186 malloc_mutex_lock(&huge_mtx);
187
188 /* Extract from tree of huge allocations. */
189 key.addr = __DECONST(void *, ptr);
190 node = extent_tree_ad_search(&huge, &key);
191 assert(node != NULL);
192
193 ret = node->size;
194
195 malloc_mutex_unlock(&huge_mtx);
196
197 return (ret);
198 }
199
200 prof_ctx_t *
huge_prof_ctx_get(const void * ptr)201 huge_prof_ctx_get(const void *ptr)
202 {
203 prof_ctx_t *ret;
204 extent_node_t *node, key;
205
206 malloc_mutex_lock(&huge_mtx);
207
208 /* Extract from tree of huge allocations. */
209 key.addr = __DECONST(void *, ptr);
210 node = extent_tree_ad_search(&huge, &key);
211 assert(node != NULL);
212
213 ret = node->prof_ctx;
214
215 malloc_mutex_unlock(&huge_mtx);
216
217 return (ret);
218 }
219
220 void
huge_prof_ctx_set(const void * ptr,prof_ctx_t * ctx)221 huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
222 {
223 extent_node_t *node, key;
224
225 malloc_mutex_lock(&huge_mtx);
226
227 /* Extract from tree of huge allocations. */
228 key.addr = __DECONST(void *, ptr);
229 node = extent_tree_ad_search(&huge, &key);
230 assert(node != NULL);
231
232 node->prof_ctx = ctx;
233
234 malloc_mutex_unlock(&huge_mtx);
235 }
236
237 bool
huge_boot(void)238 huge_boot(void)
239 {
240
241 /* Initialize chunks data. */
242 if (malloc_mutex_init(&huge_mtx))
243 return (true);
244 extent_tree_ad_new(&huge);
245
246 return (false);
247 }
248
249 void
huge_prefork(void)250 huge_prefork(void)
251 {
252
253 malloc_mutex_prefork(&huge_mtx);
254 }
255
256 void
huge_postfork_parent(void)257 huge_postfork_parent(void)
258 {
259
260 malloc_mutex_postfork_parent(&huge_mtx);
261 }
262
263 void
huge_postfork_child(void)264 huge_postfork_child(void)
265 {
266
267 malloc_mutex_postfork_child(&huge_mtx);
268 }
269