1 #ifndef Py_ATOMIC_H
2 #define Py_ATOMIC_H
3 #ifdef Py_BUILD_CORE
4
5 #include "dynamic_annotations.h"
6
7 #include "pyconfig.h"
8
9 #if defined(HAVE_STD_ATOMIC)
10 #include <stdatomic.h>
11 #endif
12
13 /* This is modeled after the atomics interface from C1x, according to
14 * the draft at
15 * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf.
16 * Operations and types are named the same except with a _Py_ prefix
17 * and have the same semantics.
18 *
19 * Beware, the implementations here are deep magic.
20 */
21
22 #if defined(HAVE_STD_ATOMIC)
23
24 typedef enum _Py_memory_order {
25 _Py_memory_order_relaxed = memory_order_relaxed,
26 _Py_memory_order_acquire = memory_order_acquire,
27 _Py_memory_order_release = memory_order_release,
28 _Py_memory_order_acq_rel = memory_order_acq_rel,
29 _Py_memory_order_seq_cst = memory_order_seq_cst
30 } _Py_memory_order;
31
32 typedef struct _Py_atomic_address {
33 atomic_uintptr_t _value;
34 } _Py_atomic_address;
35
36 typedef struct _Py_atomic_int {
37 atomic_int _value;
38 } _Py_atomic_int;
39
40 #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
41 atomic_signal_fence(ORDER)
42
43 #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
44 atomic_thread_fence(ORDER)
45
46 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
47 atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
48
49 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
50 atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
51
52 /* Use builtin atomic operations in GCC >= 4.7 */
53 #elif defined(HAVE_BUILTIN_ATOMIC)
54
55 typedef enum _Py_memory_order {
56 _Py_memory_order_relaxed = __ATOMIC_RELAXED,
57 _Py_memory_order_acquire = __ATOMIC_ACQUIRE,
58 _Py_memory_order_release = __ATOMIC_RELEASE,
59 _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL,
60 _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST
61 } _Py_memory_order;
62
63 typedef struct _Py_atomic_address {
64 uintptr_t _value;
65 } _Py_atomic_address;
66
67 typedef struct _Py_atomic_int {
68 int _value;
69 } _Py_atomic_int;
70
71 #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
72 __atomic_signal_fence(ORDER)
73
74 #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
75 __atomic_thread_fence(ORDER)
76
77 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
78 (assert((ORDER) == __ATOMIC_RELAXED \
79 || (ORDER) == __ATOMIC_SEQ_CST \
80 || (ORDER) == __ATOMIC_RELEASE), \
81 __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER))
82
83 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
84 (assert((ORDER) == __ATOMIC_RELAXED \
85 || (ORDER) == __ATOMIC_SEQ_CST \
86 || (ORDER) == __ATOMIC_ACQUIRE \
87 || (ORDER) == __ATOMIC_CONSUME), \
88 __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER))
89
90 #else
91
92 typedef enum _Py_memory_order {
93 _Py_memory_order_relaxed,
94 _Py_memory_order_acquire,
95 _Py_memory_order_release,
96 _Py_memory_order_acq_rel,
97 _Py_memory_order_seq_cst
98 } _Py_memory_order;
99
100 typedef struct _Py_atomic_address {
101 uintptr_t _value;
102 } _Py_atomic_address;
103
104 typedef struct _Py_atomic_int {
105 int _value;
106 } _Py_atomic_int;
107
108 /* Only support GCC (for expression statements) and x86 (for simple
109 * atomic semantics) for now */
110 #if defined(__GNUC__) && (defined(__i386__) || defined(__amd64))
111
112 static __inline__ void
_Py_atomic_signal_fence(_Py_memory_order order)113 _Py_atomic_signal_fence(_Py_memory_order order)
114 {
115 if (order != _Py_memory_order_relaxed)
116 __asm__ volatile("":::"memory");
117 }
118
119 static __inline__ void
_Py_atomic_thread_fence(_Py_memory_order order)120 _Py_atomic_thread_fence(_Py_memory_order order)
121 {
122 if (order != _Py_memory_order_relaxed)
123 __asm__ volatile("mfence":::"memory");
124 }
125
126 /* Tell the race checker about this operation's effects. */
127 static __inline__ void
_Py_ANNOTATE_MEMORY_ORDER(const volatile void * address,_Py_memory_order order)128 _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
129 {
130 (void)address; /* shut up -Wunused-parameter */
131 switch(order) {
132 case _Py_memory_order_release:
133 case _Py_memory_order_acq_rel:
134 case _Py_memory_order_seq_cst:
135 _Py_ANNOTATE_HAPPENS_BEFORE(address);
136 break;
137 case _Py_memory_order_relaxed:
138 case _Py_memory_order_acquire:
139 break;
140 }
141 switch(order) {
142 case _Py_memory_order_acquire:
143 case _Py_memory_order_acq_rel:
144 case _Py_memory_order_seq_cst:
145 _Py_ANNOTATE_HAPPENS_AFTER(address);
146 break;
147 case _Py_memory_order_relaxed:
148 case _Py_memory_order_release:
149 break;
150 }
151 }
152
153 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
154 __extension__ ({ \
155 __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
156 __typeof__(atomic_val->_value) new_val = NEW_VAL;\
157 volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \
158 _Py_memory_order order = ORDER; \
159 _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
160 \
161 /* Perform the operation. */ \
162 _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \
163 switch(order) { \
164 case _Py_memory_order_release: \
165 _Py_atomic_signal_fence(_Py_memory_order_release); \
166 /* fallthrough */ \
167 case _Py_memory_order_relaxed: \
168 *volatile_data = new_val; \
169 break; \
170 \
171 case _Py_memory_order_acquire: \
172 case _Py_memory_order_acq_rel: \
173 case _Py_memory_order_seq_cst: \
174 __asm__ volatile("xchg %0, %1" \
175 : "+r"(new_val) \
176 : "m"(atomic_val->_value) \
177 : "memory"); \
178 break; \
179 } \
180 _Py_ANNOTATE_IGNORE_WRITES_END(); \
181 })
182
183 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
184 __extension__ ({ \
185 __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
186 __typeof__(atomic_val->_value) result; \
187 volatile __typeof__(result) *volatile_data = &atomic_val->_value; \
188 _Py_memory_order order = ORDER; \
189 _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
190 \
191 /* Perform the operation. */ \
192 _Py_ANNOTATE_IGNORE_READS_BEGIN(); \
193 switch(order) { \
194 case _Py_memory_order_release: \
195 case _Py_memory_order_acq_rel: \
196 case _Py_memory_order_seq_cst: \
197 /* Loads on x86 are not releases by default, so need a */ \
198 /* thread fence. */ \
199 _Py_atomic_thread_fence(_Py_memory_order_release); \
200 break; \
201 default: \
202 /* No fence */ \
203 break; \
204 } \
205 result = *volatile_data; \
206 switch(order) { \
207 case _Py_memory_order_acquire: \
208 case _Py_memory_order_acq_rel: \
209 case _Py_memory_order_seq_cst: \
210 /* Loads on x86 are automatically acquire operations so */ \
211 /* can get by with just a compiler fence. */ \
212 _Py_atomic_signal_fence(_Py_memory_order_acquire); \
213 break; \
214 default: \
215 /* No fence */ \
216 break; \
217 } \
218 _Py_ANNOTATE_IGNORE_READS_END(); \
219 result; \
220 })
221
222 #else /* !gcc x86 */
223 /* Fall back to other compilers and processors by assuming that simple
224 volatile accesses are atomic. This is false, so people should port
225 this. */
226 #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0)
227 #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0)
228 #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
229 ((ATOMIC_VAL)->_value = NEW_VAL)
230 #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
231 ((ATOMIC_VAL)->_value)
232
233 #endif /* !gcc x86 */
234 #endif
235
236 /* Standardized shortcuts. */
237 #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \
238 _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst)
239 #define _Py_atomic_load(ATOMIC_VAL) \
240 _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst)
241
242 /* Python-local extensions */
243
244 #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \
245 _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed)
246 #define _Py_atomic_load_relaxed(ATOMIC_VAL) \
247 _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed)
248
249 #endif /* Py_BUILD_CORE */
250 #endif /* Py_ATOMIC_H */
251