1 #define JEMALLOC_CHUNK_MMAP_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /******************************************************************************/
5 /* Defines/includes needed for special android code. */
6
7 #if defined(__ANDROID__)
8 #include <sys/prctl.h>
9
10 /* Definitions of prctl arguments to set a vma name in Android kernels. */
11 #define ANDROID_PR_SET_VMA 0x53564d41
12 #define ANDROID_PR_SET_VMA_ANON_NAME 0
13 #endif
14
15 /******************************************************************************/
16
17 /******************************************************************************/
18 /* Function prototypes for non-inline static functions. */
19
20 static void *pages_map(void *addr, size_t size);
21 static void pages_unmap(void *addr, size_t size);
22 static void *chunk_alloc_mmap_slow(size_t size, size_t alignment,
23 bool *zero);
24
25 /******************************************************************************/
26
27 static void *
pages_map(void * addr,size_t size)28 pages_map(void *addr, size_t size)
29 {
30 void *ret;
31
32 assert(size != 0);
33
34 #ifdef _WIN32
35 /*
36 * If VirtualAlloc can't allocate at the given address when one is
37 * given, it fails and returns NULL.
38 */
39 ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
40 PAGE_READWRITE);
41 #else
42 /*
43 * We don't use MAP_FIXED here, because it can cause the *replacement*
44 * of existing mappings, and we only want to create new mappings.
45 */
46 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
47 -1, 0);
48 assert(ret != NULL);
49
50 if (ret == MAP_FAILED)
51 ret = NULL;
52 else if (addr != NULL && ret != addr) {
53 /*
54 * We succeeded in mapping memory, but not in the right place.
55 */
56 if (munmap(ret, size) == -1) {
57 char buf[BUFERROR_BUF];
58
59 buferror(get_errno(), buf, sizeof(buf));
60 malloc_printf("<jemalloc: Error in munmap(): %s\n",
61 buf);
62 if (opt_abort)
63 abort();
64 }
65 ret = NULL;
66 }
67 #endif
68 #if defined(__ANDROID__)
69 if (ret != NULL) {
70 /* Name this memory as being used by libc */
71 prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, ret,
72 size, "libc_malloc");
73 }
74 #endif
75 assert(ret == NULL || (addr == NULL && ret != addr)
76 || (addr != NULL && ret == addr));
77 return (ret);
78 }
79
80 static void
pages_unmap(void * addr,size_t size)81 pages_unmap(void *addr, size_t size)
82 {
83
84 #ifdef _WIN32
85 if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
86 #else
87 if (munmap(addr, size) == -1)
88 #endif
89 {
90 char buf[BUFERROR_BUF];
91
92 buferror(get_errno(), buf, sizeof(buf));
93 malloc_printf("<jemalloc>: Error in "
94 #ifdef _WIN32
95 "VirtualFree"
96 #else
97 "munmap"
98 #endif
99 "(): %s\n", buf);
100 if (opt_abort)
101 abort();
102 }
103 }
104
105 static void *
pages_trim(void * addr,size_t alloc_size,size_t leadsize,size_t size)106 pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
107 {
108 void *ret = (void *)((uintptr_t)addr + leadsize);
109
110 assert(alloc_size >= leadsize + size);
111 #ifdef _WIN32
112 {
113 void *new_addr;
114
115 pages_unmap(addr, alloc_size);
116 new_addr = pages_map(ret, size);
117 if (new_addr == ret)
118 return (ret);
119 if (new_addr)
120 pages_unmap(new_addr, size);
121 return (NULL);
122 }
123 #else
124 {
125 size_t trailsize = alloc_size - leadsize - size;
126
127 if (leadsize != 0)
128 pages_unmap(addr, leadsize);
129 if (trailsize != 0)
130 pages_unmap((void *)((uintptr_t)ret + size), trailsize);
131 return (ret);
132 }
133 #endif
134 }
135
136 bool
pages_purge(void * addr,size_t length)137 pages_purge(void *addr, size_t length)
138 {
139 bool unzeroed;
140
141 #ifdef _WIN32
142 VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
143 unzeroed = true;
144 #elif defined(JEMALLOC_HAVE_MADVISE)
145 # ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
146 # define JEMALLOC_MADV_PURGE MADV_DONTNEED
147 # define JEMALLOC_MADV_ZEROS true
148 # elif defined(JEMALLOC_PURGE_MADVISE_FREE)
149 # define JEMALLOC_MADV_PURGE MADV_FREE
150 # define JEMALLOC_MADV_ZEROS false
151 # else
152 # error "No madvise(2) flag defined for purging unused dirty pages."
153 # endif
154 int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
155 unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0);
156 # undef JEMALLOC_MADV_PURGE
157 # undef JEMALLOC_MADV_ZEROS
158 #else
159 /* Last resort no-op. */
160 unzeroed = true;
161 #endif
162 return (unzeroed);
163 }
164
165 static void *
chunk_alloc_mmap_slow(size_t size,size_t alignment,bool * zero)166 chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero)
167 {
168 void *ret, *pages;
169 size_t alloc_size, leadsize;
170
171 alloc_size = size + alignment - PAGE;
172 /* Beware size_t wrap-around. */
173 if (alloc_size < size)
174 return (NULL);
175 do {
176 pages = pages_map(NULL, alloc_size);
177 if (pages == NULL)
178 return (NULL);
179 leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
180 (uintptr_t)pages;
181 ret = pages_trim(pages, alloc_size, leadsize, size);
182 } while (ret == NULL);
183
184 assert(ret != NULL);
185 *zero = true;
186 return (ret);
187 }
188
189 void *
chunk_alloc_mmap(size_t size,size_t alignment,bool * zero)190 chunk_alloc_mmap(size_t size, size_t alignment, bool *zero)
191 {
192 void *ret;
193 size_t offset;
194
195 /*
196 * Ideally, there would be a way to specify alignment to mmap() (like
197 * NetBSD has), but in the absence of such a feature, we have to work
198 * hard to efficiently create aligned mappings. The reliable, but
199 * slow method is to create a mapping that is over-sized, then trim the
200 * excess. However, that always results in one or two calls to
201 * pages_unmap().
202 *
203 * Optimistically try mapping precisely the right amount before falling
204 * back to the slow method, with the expectation that the optimistic
205 * approach works most of the time.
206 */
207
208 assert(alignment != 0);
209 assert((alignment & chunksize_mask) == 0);
210
211 ret = pages_map(NULL, size);
212 if (ret == NULL)
213 return (NULL);
214 offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
215 if (offset != 0) {
216 pages_unmap(ret, size);
217 return (chunk_alloc_mmap_slow(size, alignment, zero));
218 }
219
220 assert(ret != NULL);
221 *zero = true;
222 return (ret);
223 }
224
225 bool
chunk_dalloc_mmap(void * chunk,size_t size)226 chunk_dalloc_mmap(void *chunk, size_t size)
227 {
228
229 if (config_munmap)
230 pages_unmap(chunk, size);
231
232 return (config_munmap == false);
233 }
234