1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 #endif /* JEMALLOC_H_TYPES */
5 /******************************************************************************/
6 #ifdef JEMALLOC_H_STRUCTS
7
8 #endif /* JEMALLOC_H_STRUCTS */
9 /******************************************************************************/
10 #ifdef JEMALLOC_H_EXTERNS
11
12 #define atomic_read_uint64(p) atomic_add_uint64(p, 0)
13 #define atomic_read_uint32(p) atomic_add_uint32(p, 0)
14 #define atomic_read_p(p) atomic_add_p(p, NULL)
15 #define atomic_read_z(p) atomic_add_z(p, 0)
16 #define atomic_read_u(p) atomic_add_u(p, 0)
17
18 #endif /* JEMALLOC_H_EXTERNS */
19 /******************************************************************************/
20 #ifdef JEMALLOC_H_INLINES
21
22 /*
23 * All arithmetic functions return the arithmetic result of the atomic
24 * operation. Some atomic operation APIs return the value prior to mutation, in
25 * which case the following functions must redundantly compute the result so
26 * that it can be returned. These functions are normally inlined, so the extra
27 * operations can be optimized away if the return values aren't used by the
28 * callers.
29 *
30 * <t> atomic_read_<t>(<t> *p) { return (*p); }
31 * <t> atomic_add_<t>(<t> *p, <t> x) { return (*p += x); }
32 * <t> atomic_sub_<t>(<t> *p, <t> x) { return (*p -= x); }
33 * bool atomic_cas_<t>(<t> *p, <t> c, <t> s)
34 * {
35 * if (*p != c)
36 * return (true);
37 * *p = s;
38 * return (false);
39 * }
40 * void atomic_write_<t>(<t> *p, <t> x) { *p = x; }
41 */
42
43 #ifndef JEMALLOC_ENABLE_INLINE
44 uint64_t atomic_add_uint64(uint64_t *p, uint64_t x);
45 uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x);
46 bool atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s);
47 void atomic_write_uint64(uint64_t *p, uint64_t x);
48 uint32_t atomic_add_uint32(uint32_t *p, uint32_t x);
49 uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x);
50 bool atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s);
51 void atomic_write_uint32(uint32_t *p, uint32_t x);
52 void *atomic_add_p(void **p, void *x);
53 void *atomic_sub_p(void **p, void *x);
54 bool atomic_cas_p(void **p, void *c, void *s);
55 void atomic_write_p(void **p, const void *x);
56 size_t atomic_add_z(size_t *p, size_t x);
57 size_t atomic_sub_z(size_t *p, size_t x);
58 bool atomic_cas_z(size_t *p, size_t c, size_t s);
59 void atomic_write_z(size_t *p, size_t x);
60 unsigned atomic_add_u(unsigned *p, unsigned x);
61 unsigned atomic_sub_u(unsigned *p, unsigned x);
62 bool atomic_cas_u(unsigned *p, unsigned c, unsigned s);
63 void atomic_write_u(unsigned *p, unsigned x);
64 #endif
65
66 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
67 /******************************************************************************/
68 /* 64-bit operations. */
69 #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
70 # if (defined(__amd64__) || defined(__x86_64__))
71 JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t * p,uint64_t x)72 atomic_add_uint64(uint64_t *p, uint64_t x)
73 {
74 uint64_t t = x;
75
76 asm volatile (
77 "lock; xaddq %0, %1;"
78 : "+r" (t), "=m" (*p) /* Outputs. */
79 : "m" (*p) /* Inputs. */
80 );
81
82 return (t + x);
83 }
84
85 JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t * p,uint64_t x)86 atomic_sub_uint64(uint64_t *p, uint64_t x)
87 {
88 uint64_t t;
89
90 x = (uint64_t)(-(int64_t)x);
91 t = x;
92 asm volatile (
93 "lock; xaddq %0, %1;"
94 : "+r" (t), "=m" (*p) /* Outputs. */
95 : "m" (*p) /* Inputs. */
96 );
97
98 return (t + x);
99 }
100
101 JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t * p,uint64_t c,uint64_t s)102 atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
103 {
104 uint8_t success;
105
106 asm volatile (
107 "lock; cmpxchgq %4, %0;"
108 "sete %1;"
109 : "=m" (*p), "=a" (success) /* Outputs. */
110 : "m" (*p), "a" (c), "r" (s) /* Inputs. */
111 : "memory" /* Clobbers. */
112 );
113
114 return (!(bool)success);
115 }
116
117 JEMALLOC_INLINE void
atomic_write_uint64(uint64_t * p,uint64_t x)118 atomic_write_uint64(uint64_t *p, uint64_t x)
119 {
120
121 asm volatile (
122 "xchgq %1, %0;" /* Lock is implied by xchgq. */
123 : "=m" (*p), "+r" (x) /* Outputs. */
124 : "m" (*p) /* Inputs. */
125 : "memory" /* Clobbers. */
126 );
127 }
128 # elif (defined(JEMALLOC_C11ATOMICS))
129 JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t * p,uint64_t x)130 atomic_add_uint64(uint64_t *p, uint64_t x)
131 {
132 volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
133 return (atomic_fetch_add(a, x) + x);
134 }
135
136 JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t * p,uint64_t x)137 atomic_sub_uint64(uint64_t *p, uint64_t x)
138 {
139 volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
140 return (atomic_fetch_sub(a, x) - x);
141 }
142
143 JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t * p,uint64_t c,uint64_t s)144 atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
145 {
146 volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
147 return (!atomic_compare_exchange_strong(a, &c, s));
148 }
149
150 JEMALLOC_INLINE void
atomic_write_uint64(uint64_t * p,uint64_t x)151 atomic_write_uint64(uint64_t *p, uint64_t x)
152 {
153 volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
154 atomic_store(a, x);
155 }
156 # elif (defined(JEMALLOC_ATOMIC9))
157 JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t * p,uint64_t x)158 atomic_add_uint64(uint64_t *p, uint64_t x)
159 {
160
161 /*
162 * atomic_fetchadd_64() doesn't exist, but we only ever use this
163 * function on LP64 systems, so atomic_fetchadd_long() will do.
164 */
165 assert(sizeof(uint64_t) == sizeof(unsigned long));
166
167 return (atomic_fetchadd_long(p, (unsigned long)x) + x);
168 }
169
170 JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t * p,uint64_t x)171 atomic_sub_uint64(uint64_t *p, uint64_t x)
172 {
173
174 assert(sizeof(uint64_t) == sizeof(unsigned long));
175
176 return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
177 }
178
179 JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t * p,uint64_t c,uint64_t s)180 atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
181 {
182
183 assert(sizeof(uint64_t) == sizeof(unsigned long));
184
185 return (!atomic_cmpset_long(p, (unsigned long)c, (unsigned long)s));
186 }
187
188 JEMALLOC_INLINE void
atomic_write_uint64(uint64_t * p,uint64_t x)189 atomic_write_uint64(uint64_t *p, uint64_t x)
190 {
191
192 assert(sizeof(uint64_t) == sizeof(unsigned long));
193
194 atomic_store_rel_long(p, x);
195 }
196 # elif (defined(JEMALLOC_OSATOMIC))
197 JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t * p,uint64_t x)198 atomic_add_uint64(uint64_t *p, uint64_t x)
199 {
200
201 return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
202 }
203
204 JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t * p,uint64_t x)205 atomic_sub_uint64(uint64_t *p, uint64_t x)
206 {
207
208 return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
209 }
210
211 JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t * p,uint64_t c,uint64_t s)212 atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
213 {
214
215 return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p));
216 }
217
218 JEMALLOC_INLINE void
atomic_write_uint64(uint64_t * p,uint64_t x)219 atomic_write_uint64(uint64_t *p, uint64_t x)
220 {
221 uint64_t o;
222
223 /*The documented OSAtomic*() API does not expose an atomic exchange. */
224 do {
225 o = atomic_read_uint64(p);
226 } while (atomic_cas_uint64(p, o, x));
227 }
228 # elif (defined(_MSC_VER))
229 JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t * p,uint64_t x)230 atomic_add_uint64(uint64_t *p, uint64_t x)
231 {
232
233 return (InterlockedExchangeAdd64(p, x) + x);
234 }
235
236 JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t * p,uint64_t x)237 atomic_sub_uint64(uint64_t *p, uint64_t x)
238 {
239
240 return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x);
241 }
242
243 JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t * p,uint64_t c,uint64_t s)244 atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
245 {
246 uint64_t o;
247
248 o = InterlockedCompareExchange64(p, s, c);
249 return (o != c);
250 }
251
252 JEMALLOC_INLINE void
atomic_write_uint64(uint64_t * p,uint64_t x)253 atomic_write_uint64(uint64_t *p, uint64_t x)
254 {
255
256 InterlockedExchange64(p, x);
257 }
258 # elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \
259 defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
260 JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t * p,uint64_t x)261 atomic_add_uint64(uint64_t *p, uint64_t x)
262 {
263
264 return (__sync_add_and_fetch(p, x));
265 }
266
267 JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t * p,uint64_t x)268 atomic_sub_uint64(uint64_t *p, uint64_t x)
269 {
270
271 return (__sync_sub_and_fetch(p, x));
272 }
273
274 JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t * p,uint64_t c,uint64_t s)275 atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
276 {
277
278 return (!__sync_bool_compare_and_swap(p, c, s));
279 }
280
281 JEMALLOC_INLINE void
atomic_write_uint64(uint64_t * p,uint64_t x)282 atomic_write_uint64(uint64_t *p, uint64_t x)
283 {
284
285 __sync_lock_test_and_set(p, x);
286 }
287 # else
288 # error "Missing implementation for 64-bit atomic operations"
289 # endif
290 #endif
291
292 /******************************************************************************/
293 /* 32-bit operations. */
294 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
295 JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t * p,uint32_t x)296 atomic_add_uint32(uint32_t *p, uint32_t x)
297 {
298 uint32_t t = x;
299
300 asm volatile (
301 "lock; xaddl %0, %1;"
302 : "+r" (t), "=m" (*p) /* Outputs. */
303 : "m" (*p) /* Inputs. */
304 );
305
306 return (t + x);
307 }
308
309 JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t * p,uint32_t x)310 atomic_sub_uint32(uint32_t *p, uint32_t x)
311 {
312 uint32_t t;
313
314 x = (uint32_t)(-(int32_t)x);
315 t = x;
316 asm volatile (
317 "lock; xaddl %0, %1;"
318 : "+r" (t), "=m" (*p) /* Outputs. */
319 : "m" (*p) /* Inputs. */
320 );
321
322 return (t + x);
323 }
324
325 JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t * p,uint32_t c,uint32_t s)326 atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
327 {
328 uint8_t success;
329
330 asm volatile (
331 "lock; cmpxchgl %4, %0;"
332 "sete %1;"
333 : "=m" (*p), "=a" (success) /* Outputs. */
334 : "m" (*p), "a" (c), "r" (s) /* Inputs. */
335 : "memory"
336 );
337
338 return (!(bool)success);
339 }
340
341 JEMALLOC_INLINE void
atomic_write_uint32(uint32_t * p,uint32_t x)342 atomic_write_uint32(uint32_t *p, uint32_t x)
343 {
344
345 asm volatile (
346 "xchgl %1, %0;" /* Lock is implied by xchgl. */
347 : "=m" (*p), "+r" (x) /* Outputs. */
348 : "m" (*p) /* Inputs. */
349 : "memory" /* Clobbers. */
350 );
351 }
352 # elif (defined(JEMALLOC_C11ATOMICS))
353 JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t * p,uint32_t x)354 atomic_add_uint32(uint32_t *p, uint32_t x)
355 {
356 volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
357 return (atomic_fetch_add(a, x) + x);
358 }
359
360 JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t * p,uint32_t x)361 atomic_sub_uint32(uint32_t *p, uint32_t x)
362 {
363 volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
364 return (atomic_fetch_sub(a, x) - x);
365 }
366
367 JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t * p,uint32_t c,uint32_t s)368 atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
369 {
370 volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
371 return (!atomic_compare_exchange_strong(a, &c, s));
372 }
373
374 JEMALLOC_INLINE void
atomic_write_uint32(uint32_t * p,uint32_t x)375 atomic_write_uint32(uint32_t *p, uint32_t x)
376 {
377 volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
378 atomic_store(a, x);
379 }
380 #elif (defined(JEMALLOC_ATOMIC9))
381 JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t * p,uint32_t x)382 atomic_add_uint32(uint32_t *p, uint32_t x)
383 {
384
385 return (atomic_fetchadd_32(p, x) + x);
386 }
387
388 JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t * p,uint32_t x)389 atomic_sub_uint32(uint32_t *p, uint32_t x)
390 {
391
392 return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
393 }
394
395 JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t * p,uint32_t c,uint32_t s)396 atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
397 {
398
399 return (!atomic_cmpset_32(p, c, s));
400 }
401
402 JEMALLOC_INLINE void
atomic_write_uint32(uint32_t * p,uint32_t x)403 atomic_write_uint32(uint32_t *p, uint32_t x)
404 {
405
406 atomic_store_rel_32(p, x);
407 }
408 #elif (defined(JEMALLOC_OSATOMIC))
409 JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t * p,uint32_t x)410 atomic_add_uint32(uint32_t *p, uint32_t x)
411 {
412
413 return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
414 }
415
416 JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t * p,uint32_t x)417 atomic_sub_uint32(uint32_t *p, uint32_t x)
418 {
419
420 return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
421 }
422
423 JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t * p,uint32_t c,uint32_t s)424 atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
425 {
426
427 return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p));
428 }
429
430 JEMALLOC_INLINE void
atomic_write_uint32(uint32_t * p,uint32_t x)431 atomic_write_uint32(uint32_t *p, uint32_t x)
432 {
433 uint32_t o;
434
435 /*The documented OSAtomic*() API does not expose an atomic exchange. */
436 do {
437 o = atomic_read_uint32(p);
438 } while (atomic_cas_uint32(p, o, x));
439 }
440 #elif (defined(_MSC_VER))
441 JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t * p,uint32_t x)442 atomic_add_uint32(uint32_t *p, uint32_t x)
443 {
444
445 return (InterlockedExchangeAdd(p, x) + x);
446 }
447
448 JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t * p,uint32_t x)449 atomic_sub_uint32(uint32_t *p, uint32_t x)
450 {
451
452 return (InterlockedExchangeAdd(p, -((int32_t)x)) - x);
453 }
454
455 JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t * p,uint32_t c,uint32_t s)456 atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
457 {
458 uint32_t o;
459
460 o = InterlockedCompareExchange(p, s, c);
461 return (o != c);
462 }
463
464 JEMALLOC_INLINE void
atomic_write_uint32(uint32_t * p,uint32_t x)465 atomic_write_uint32(uint32_t *p, uint32_t x)
466 {
467
468 InterlockedExchange(p, x);
469 }
470 #elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \
471 defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
472 JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t * p,uint32_t x)473 atomic_add_uint32(uint32_t *p, uint32_t x)
474 {
475
476 return (__sync_add_and_fetch(p, x));
477 }
478
479 JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t * p,uint32_t x)480 atomic_sub_uint32(uint32_t *p, uint32_t x)
481 {
482
483 return (__sync_sub_and_fetch(p, x));
484 }
485
486 JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t * p,uint32_t c,uint32_t s)487 atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
488 {
489
490 return (!__sync_bool_compare_and_swap(p, c, s));
491 }
492
493 JEMALLOC_INLINE void
atomic_write_uint32(uint32_t * p,uint32_t x)494 atomic_write_uint32(uint32_t *p, uint32_t x)
495 {
496
497 __sync_lock_test_and_set(p, x);
498 }
499 #else
500 # error "Missing implementation for 32-bit atomic operations"
501 #endif
502
503 /******************************************************************************/
504 /* Pointer operations. */
505 JEMALLOC_INLINE void *
atomic_add_p(void ** p,void * x)506 atomic_add_p(void **p, void *x)
507 {
508
509 #if (LG_SIZEOF_PTR == 3)
510 return ((void *)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
511 #elif (LG_SIZEOF_PTR == 2)
512 return ((void *)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
513 #endif
514 }
515
516 JEMALLOC_INLINE void *
atomic_sub_p(void ** p,void * x)517 atomic_sub_p(void **p, void *x)
518 {
519
520 #if (LG_SIZEOF_PTR == 3)
521 return ((void *)atomic_add_uint64((uint64_t *)p,
522 (uint64_t)-((int64_t)x)));
523 #elif (LG_SIZEOF_PTR == 2)
524 return ((void *)atomic_add_uint32((uint32_t *)p,
525 (uint32_t)-((int32_t)x)));
526 #endif
527 }
528
529 JEMALLOC_INLINE bool
atomic_cas_p(void ** p,void * c,void * s)530 atomic_cas_p(void **p, void *c, void *s)
531 {
532
533 #if (LG_SIZEOF_PTR == 3)
534 return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
535 #elif (LG_SIZEOF_PTR == 2)
536 return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
537 #endif
538 }
539
540 JEMALLOC_INLINE void
atomic_write_p(void ** p,const void * x)541 atomic_write_p(void **p, const void *x)
542 {
543
544 #if (LG_SIZEOF_PTR == 3)
545 atomic_write_uint64((uint64_t *)p, (uint64_t)x);
546 #elif (LG_SIZEOF_PTR == 2)
547 atomic_write_uint32((uint32_t *)p, (uint32_t)x);
548 #endif
549 }
550
551 /******************************************************************************/
552 /* size_t operations. */
553 JEMALLOC_INLINE size_t
atomic_add_z(size_t * p,size_t x)554 atomic_add_z(size_t *p, size_t x)
555 {
556
557 #if (LG_SIZEOF_PTR == 3)
558 return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
559 #elif (LG_SIZEOF_PTR == 2)
560 return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
561 #endif
562 }
563
564 JEMALLOC_INLINE size_t
atomic_sub_z(size_t * p,size_t x)565 atomic_sub_z(size_t *p, size_t x)
566 {
567
568 #if (LG_SIZEOF_PTR == 3)
569 return ((size_t)atomic_add_uint64((uint64_t *)p,
570 (uint64_t)-((int64_t)x)));
571 #elif (LG_SIZEOF_PTR == 2)
572 return ((size_t)atomic_add_uint32((uint32_t *)p,
573 (uint32_t)-((int32_t)x)));
574 #endif
575 }
576
577 JEMALLOC_INLINE bool
atomic_cas_z(size_t * p,size_t c,size_t s)578 atomic_cas_z(size_t *p, size_t c, size_t s)
579 {
580
581 #if (LG_SIZEOF_PTR == 3)
582 return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
583 #elif (LG_SIZEOF_PTR == 2)
584 return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
585 #endif
586 }
587
588 JEMALLOC_INLINE void
atomic_write_z(size_t * p,size_t x)589 atomic_write_z(size_t *p, size_t x)
590 {
591
592 #if (LG_SIZEOF_PTR == 3)
593 atomic_write_uint64((uint64_t *)p, (uint64_t)x);
594 #elif (LG_SIZEOF_PTR == 2)
595 atomic_write_uint32((uint32_t *)p, (uint32_t)x);
596 #endif
597 }
598
599 /******************************************************************************/
600 /* unsigned operations. */
601 JEMALLOC_INLINE unsigned
atomic_add_u(unsigned * p,unsigned x)602 atomic_add_u(unsigned *p, unsigned x)
603 {
604
605 #if (LG_SIZEOF_INT == 3)
606 return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
607 #elif (LG_SIZEOF_INT == 2)
608 return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
609 #endif
610 }
611
612 JEMALLOC_INLINE unsigned
atomic_sub_u(unsigned * p,unsigned x)613 atomic_sub_u(unsigned *p, unsigned x)
614 {
615
616 #if (LG_SIZEOF_INT == 3)
617 return ((unsigned)atomic_add_uint64((uint64_t *)p,
618 (uint64_t)-((int64_t)x)));
619 #elif (LG_SIZEOF_INT == 2)
620 return ((unsigned)atomic_add_uint32((uint32_t *)p,
621 (uint32_t)-((int32_t)x)));
622 #endif
623 }
624
625 JEMALLOC_INLINE bool
atomic_cas_u(unsigned * p,unsigned c,unsigned s)626 atomic_cas_u(unsigned *p, unsigned c, unsigned s)
627 {
628
629 #if (LG_SIZEOF_INT == 3)
630 return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
631 #elif (LG_SIZEOF_INT == 2)
632 return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
633 #endif
634 }
635
636 JEMALLOC_INLINE void
atomic_write_u(unsigned * p,unsigned x)637 atomic_write_u(unsigned *p, unsigned x)
638 {
639
640 #if (LG_SIZEOF_INT == 3)
641 atomic_write_uint64((uint64_t *)p, (uint64_t)x);
642 #elif (LG_SIZEOF_INT == 2)
643 atomic_write_uint32((uint32_t *)p, (uint32_t)x);
644 #endif
645 }
646
647 /******************************************************************************/
648 #endif
649
650 #endif /* JEMALLOC_H_INLINES */
651 /******************************************************************************/
652