1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 #ifdef _WIN32
5 # ifdef _WIN64
6 # define FMT64_PREFIX "ll"
7 # define FMTPTR_PREFIX "ll"
8 # else
9 # define FMT64_PREFIX "ll"
10 # define FMTPTR_PREFIX ""
11 # endif
12 # define FMTd32 "d"
13 # define FMTu32 "u"
14 # define FMTx32 "x"
15 # define FMTd64 FMT64_PREFIX "d"
16 # define FMTu64 FMT64_PREFIX "u"
17 # define FMTx64 FMT64_PREFIX "x"
18 # define FMTdPTR FMTPTR_PREFIX "d"
19 # define FMTuPTR FMTPTR_PREFIX "u"
20 # define FMTxPTR FMTPTR_PREFIX "x"
21 #else
22 # include <inttypes.h>
23 # define FMTd32 PRId32
24 # define FMTu32 PRIu32
25 # define FMTx32 PRIx32
26 # define FMTd64 PRId64
27 # define FMTu64 PRIu64
28 # define FMTx64 PRIx64
29 # define FMTdPTR PRIdPTR
30 # define FMTuPTR PRIuPTR
31 # define FMTxPTR PRIxPTR
32 #endif
33
34 /* Size of stack-allocated buffer passed to buferror(). */
35 #define BUFERROR_BUF 64
36
37 /*
38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be
39 * large enough for all possible uses within jemalloc.
40 */
41 #define MALLOC_PRINTF_BUFSIZE 4096
42
43 /* Junk fill patterns. */
44 #ifndef JEMALLOC_ALLOC_JUNK
45 # define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)
46 #endif
47 #ifndef JEMALLOC_FREE_JUNK
48 # define JEMALLOC_FREE_JUNK ((uint8_t)0x5a)
49 #endif
50
51 /*
52 * Wrap a cpp argument that contains commas such that it isn't broken up into
53 * multiple arguments.
54 */
55 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
56
57 /*
58 * Silence compiler warnings due to uninitialized values. This is used
59 * wherever the compiler fails to recognize that the variable is never used
60 * uninitialized.
61 */
62 #ifdef JEMALLOC_CC_SILENCE
63 # define JEMALLOC_CC_SILENCE_INIT(v) = v
64 #else
65 # define JEMALLOC_CC_SILENCE_INIT(v)
66 #endif
67
68 #ifdef __GNUC__
69 # define likely(x) __builtin_expect(!!(x), 1)
70 # define unlikely(x) __builtin_expect(!!(x), 0)
71 #else
72 # define likely(x) !!(x)
73 # define unlikely(x) !!(x)
74 #endif
75
76 #if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
77 # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
78 #endif
79
80 #define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
81
82 #include "jemalloc/internal/assert.h"
83
84 /* Use to assert a particular configuration, e.g., cassert(config_debug). */
85 #define cassert(c) do { \
86 if (unlikely(!(c))) \
87 not_reached(); \
88 } while (0)
89
90 #endif /* JEMALLOC_H_TYPES */
91 /******************************************************************************/
92 #ifdef JEMALLOC_H_STRUCTS
93
94 #endif /* JEMALLOC_H_STRUCTS */
95 /******************************************************************************/
96 #ifdef JEMALLOC_H_EXTERNS
97
98 int buferror(int err, char *buf, size_t buflen);
99 uintmax_t malloc_strtoumax(const char *restrict nptr,
100 char **restrict endptr, int base);
101 void malloc_write(const char *s);
102
103 /*
104 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
105 * point math.
106 */
107 size_t malloc_vsnprintf(char *str, size_t size, const char *format,
108 va_list ap);
109 size_t malloc_snprintf(char *str, size_t size, const char *format, ...)
110 JEMALLOC_FORMAT_PRINTF(3, 4);
111 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
112 const char *format, va_list ap);
113 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
114 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
115 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
116
117 #endif /* JEMALLOC_H_EXTERNS */
118 /******************************************************************************/
119 #ifdef JEMALLOC_H_INLINES
120
121 #ifndef JEMALLOC_ENABLE_INLINE
122 unsigned ffs_llu(unsigned long long bitmap);
123 unsigned ffs_lu(unsigned long bitmap);
124 unsigned ffs_u(unsigned bitmap);
125 unsigned ffs_zu(size_t bitmap);
126 unsigned ffs_u64(uint64_t bitmap);
127 unsigned ffs_u32(uint32_t bitmap);
128 uint64_t pow2_ceil_u64(uint64_t x);
129 uint32_t pow2_ceil_u32(uint32_t x);
130 size_t pow2_ceil_zu(size_t x);
131 unsigned lg_floor(size_t x);
132 void set_errno(int errnum);
133 int get_errno(void);
134 #endif
135
136 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
137
138 /* Sanity check. */
139 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
140 || !defined(JEMALLOC_INTERNAL_FFS)
141 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
142 #endif
143
144 JEMALLOC_ALWAYS_INLINE unsigned
ffs_llu(unsigned long long bitmap)145 ffs_llu(unsigned long long bitmap)
146 {
147
148 return (JEMALLOC_INTERNAL_FFSLL(bitmap));
149 }
150
151 JEMALLOC_ALWAYS_INLINE unsigned
ffs_lu(unsigned long bitmap)152 ffs_lu(unsigned long bitmap)
153 {
154
155 return (JEMALLOC_INTERNAL_FFSL(bitmap));
156 }
157
158 JEMALLOC_ALWAYS_INLINE unsigned
ffs_u(unsigned bitmap)159 ffs_u(unsigned bitmap)
160 {
161
162 return (JEMALLOC_INTERNAL_FFS(bitmap));
163 }
164
165 JEMALLOC_ALWAYS_INLINE unsigned
ffs_zu(size_t bitmap)166 ffs_zu(size_t bitmap)
167 {
168
169 #if LG_SIZEOF_PTR == LG_SIZEOF_INT
170 return (ffs_u(bitmap));
171 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
172 return (ffs_lu(bitmap));
173 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
174 return (ffs_llu(bitmap));
175 #else
176 #error No implementation for size_t ffs()
177 #endif
178 }
179
180 JEMALLOC_ALWAYS_INLINE unsigned
ffs_u64(uint64_t bitmap)181 ffs_u64(uint64_t bitmap)
182 {
183
184 #if LG_SIZEOF_LONG == 3
185 return (ffs_lu(bitmap));
186 #elif LG_SIZEOF_LONG_LONG == 3
187 return (ffs_llu(bitmap));
188 #else
189 #error No implementation for 64-bit ffs()
190 #endif
191 }
192
193 JEMALLOC_ALWAYS_INLINE unsigned
ffs_u32(uint32_t bitmap)194 ffs_u32(uint32_t bitmap)
195 {
196
197 #if LG_SIZEOF_INT == 2
198 return (ffs_u(bitmap));
199 #else
200 #error No implementation for 32-bit ffs()
201 #endif
202 return (ffs_u(bitmap));
203 }
204
205 JEMALLOC_INLINE uint64_t
pow2_ceil_u64(uint64_t x)206 pow2_ceil_u64(uint64_t x)
207 {
208
209 x--;
210 x |= x >> 1;
211 x |= x >> 2;
212 x |= x >> 4;
213 x |= x >> 8;
214 x |= x >> 16;
215 x |= x >> 32;
216 x++;
217 return (x);
218 }
219
220 JEMALLOC_INLINE uint32_t
pow2_ceil_u32(uint32_t x)221 pow2_ceil_u32(uint32_t x)
222 {
223
224 x--;
225 x |= x >> 1;
226 x |= x >> 2;
227 x |= x >> 4;
228 x |= x >> 8;
229 x |= x >> 16;
230 x++;
231 return (x);
232 }
233
234 /* Compute the smallest power of 2 that is >= x. */
235 JEMALLOC_INLINE size_t
pow2_ceil_zu(size_t x)236 pow2_ceil_zu(size_t x)
237 {
238
239 #if (LG_SIZEOF_PTR == 3)
240 return (pow2_ceil_u64(x));
241 #else
242 return (pow2_ceil_u32(x));
243 #endif
244 }
245
246 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
247 JEMALLOC_INLINE unsigned
lg_floor(size_t x)248 lg_floor(size_t x)
249 {
250 size_t ret;
251
252 assert(x != 0);
253
254 asm ("bsr %1, %0"
255 : "=r"(ret) // Outputs.
256 : "r"(x) // Inputs.
257 );
258 assert(ret < UINT_MAX);
259 return ((unsigned)ret);
260 }
261 #elif (defined(_MSC_VER))
262 JEMALLOC_INLINE unsigned
lg_floor(size_t x)263 lg_floor(size_t x)
264 {
265 unsigned long ret;
266
267 assert(x != 0);
268
269 #if (LG_SIZEOF_PTR == 3)
270 _BitScanReverse64(&ret, x);
271 #elif (LG_SIZEOF_PTR == 2)
272 _BitScanReverse(&ret, x);
273 #else
274 # error "Unsupported type size for lg_floor()"
275 #endif
276 assert(ret < UINT_MAX);
277 return ((unsigned)ret);
278 }
279 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
280 JEMALLOC_INLINE unsigned
lg_floor(size_t x)281 lg_floor(size_t x)
282 {
283
284 assert(x != 0);
285
286 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
287 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
288 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
289 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
290 #else
291 # error "Unsupported type size for lg_floor()"
292 #endif
293 }
294 #else
295 JEMALLOC_INLINE unsigned
lg_floor(size_t x)296 lg_floor(size_t x)
297 {
298
299 assert(x != 0);
300
301 x |= (x >> 1);
302 x |= (x >> 2);
303 x |= (x >> 4);
304 x |= (x >> 8);
305 x |= (x >> 16);
306 #if (LG_SIZEOF_PTR == 3)
307 x |= (x >> 32);
308 #endif
309 if (x == SIZE_T_MAX)
310 return ((8 << LG_SIZEOF_PTR) - 1);
311 x++;
312 return (ffs_zu(x) - 2);
313 }
314 #endif
315
316 /* Set error code. */
317 JEMALLOC_INLINE void
set_errno(int errnum)318 set_errno(int errnum)
319 {
320
321 #ifdef _WIN32
322 SetLastError(errnum);
323 #else
324 errno = errnum;
325 #endif
326 }
327
328 /* Get last error code. */
329 JEMALLOC_INLINE int
get_errno(void)330 get_errno(void)
331 {
332
333 #ifdef _WIN32
334 return (GetLastError());
335 #else
336 return (errno);
337 #endif
338 }
339 #endif
340
341 #endif /* JEMALLOC_H_INLINES */
342 /******************************************************************************/
343