• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H
2 #define JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H
3 
4 #include "jemalloc/internal/assert.h"
5 
6 #define ATOMIC_INIT(...) {__VA_ARGS__}
7 
8 typedef enum {
9 	atomic_memory_order_relaxed,
10 	atomic_memory_order_acquire,
11 	atomic_memory_order_release,
12 	atomic_memory_order_acq_rel,
13 	atomic_memory_order_seq_cst
14 } atomic_memory_order_t;
15 
16 ATOMIC_INLINE int
atomic_enum_to_builtin(atomic_memory_order_t mo)17 atomic_enum_to_builtin(atomic_memory_order_t mo) {
18 	switch (mo) {
19 	case atomic_memory_order_relaxed:
20 		return __ATOMIC_RELAXED;
21 	case atomic_memory_order_acquire:
22 		return __ATOMIC_ACQUIRE;
23 	case atomic_memory_order_release:
24 		return __ATOMIC_RELEASE;
25 	case atomic_memory_order_acq_rel:
26 		return __ATOMIC_ACQ_REL;
27 	case atomic_memory_order_seq_cst:
28 		return __ATOMIC_SEQ_CST;
29 	}
30 	/* Can't happen; the switch is exhaustive. */
31 	not_reached();
32 }
33 
34 ATOMIC_INLINE void
atomic_fence(atomic_memory_order_t mo)35 atomic_fence(atomic_memory_order_t mo) {
36 	__atomic_thread_fence(atomic_enum_to_builtin(mo));
37 }
38 
39 #define JEMALLOC_GENERATE_ATOMICS(type, short_type,			\
40     /* unused */ lg_size)						\
41 typedef struct {							\
42 	type repr;							\
43 } atomic_##short_type##_t;						\
44 									\
45 ATOMIC_INLINE type							\
46 atomic_load_##short_type(const atomic_##short_type##_t *a,		\
47     atomic_memory_order_t mo) {						\
48 	type result;							\
49 	__atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo));	\
50 	return result;							\
51 }									\
52 									\
53 ATOMIC_INLINE void							\
54 atomic_store_##short_type(atomic_##short_type##_t *a, type val,		\
55     atomic_memory_order_t mo) {						\
56 	__atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo));	\
57 }									\
58 									\
59 ATOMIC_INLINE type							\
60 atomic_exchange_##short_type(atomic_##short_type##_t *a, type val,	\
61     atomic_memory_order_t mo) {						\
62 	type result;							\
63 	__atomic_exchange(&a->repr, &val, &result,			\
64 	    atomic_enum_to_builtin(mo));				\
65 	return result;							\
66 }									\
67 									\
68 ATOMIC_INLINE bool							\
69 atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a,	\
70     type *expected, type desired, atomic_memory_order_t success_mo,	\
71     atomic_memory_order_t failure_mo) {					\
72 	return __atomic_compare_exchange(&a->repr, expected, &desired,	\
73 	    true, atomic_enum_to_builtin(success_mo),			\
74 	    atomic_enum_to_builtin(failure_mo));			\
75 }									\
76 									\
77 ATOMIC_INLINE bool							\
78 atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a,	\
79     type *expected, type desired, atomic_memory_order_t success_mo,	\
80     atomic_memory_order_t failure_mo) {					\
81 	return __atomic_compare_exchange(&a->repr, expected, &desired,	\
82 	    false,							\
83 	    atomic_enum_to_builtin(success_mo),				\
84 	    atomic_enum_to_builtin(failure_mo));			\
85 }
86 
87 
88 #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type,			\
89     /* unused */ lg_size)						\
90 JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size)	\
91 									\
92 ATOMIC_INLINE type							\
93 atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val,	\
94     atomic_memory_order_t mo) {						\
95 	return __atomic_fetch_add(&a->repr, val,			\
96 	    atomic_enum_to_builtin(mo));				\
97 }									\
98 									\
99 ATOMIC_INLINE type							\
100 atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val,	\
101     atomic_memory_order_t mo) {						\
102 	return __atomic_fetch_sub(&a->repr, val,			\
103 	    atomic_enum_to_builtin(mo));				\
104 }									\
105 									\
106 ATOMIC_INLINE type							\
107 atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val,	\
108     atomic_memory_order_t mo) {						\
109 	return __atomic_fetch_and(&a->repr, val,			\
110 	    atomic_enum_to_builtin(mo));				\
111 }									\
112 									\
113 ATOMIC_INLINE type							\
114 atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val,	\
115     atomic_memory_order_t mo) {						\
116 	return __atomic_fetch_or(&a->repr, val,				\
117 	    atomic_enum_to_builtin(mo));				\
118 }									\
119 									\
120 ATOMIC_INLINE type							\
121 atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val,	\
122     atomic_memory_order_t mo) {						\
123 	return __atomic_fetch_xor(&a->repr, val,			\
124 	    atomic_enum_to_builtin(mo));				\
125 }
126 
127 #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H */
128