1 #ifndef _ATOMIC_H
2 #define _ATOMIC_H
3
4 #include <stdint.h>
5
6 #include "atomic_arch.h"
7
8 #ifdef a_ll
9
10 #ifndef a_pre_llsc
11 #define a_pre_llsc()
12 #endif
13
14 #ifndef a_post_llsc
15 #define a_post_llsc()
16 #endif
17
18 #ifndef a_cas
19 #define a_cas a_cas
a_cas(volatile int * p,int t,int s)20 static inline int a_cas(volatile int *p, int t, int s)
21 {
22 int old;
23 a_pre_llsc();
24 do old = a_ll(p);
25 while (old==t && !a_sc(p, s));
26 a_post_llsc();
27 return old;
28 }
29 #endif
30
31 #ifndef a_swap
32 #define a_swap a_swap
a_swap(volatile int * p,int v)33 static inline int a_swap(volatile int *p, int v)
34 {
35 int old;
36 a_pre_llsc();
37 do old = a_ll(p);
38 while (!a_sc(p, v));
39 a_post_llsc();
40 return old;
41 }
42 #endif
43
44 #ifndef a_fetch_add
45 #define a_fetch_add a_fetch_add
a_fetch_add(volatile int * p,int v)46 static inline int a_fetch_add(volatile int *p, int v)
47 {
48 int old;
49 a_pre_llsc();
50 do old = a_ll(p);
51 while (!a_sc(p, (unsigned)old + v));
52 a_post_llsc();
53 return old;
54 }
55 #endif
56
57 #ifndef a_fetch_and
58 #define a_fetch_and a_fetch_and
a_fetch_and(volatile int * p,int v)59 static inline int a_fetch_and(volatile int *p, int v)
60 {
61 int old;
62 a_pre_llsc();
63 do old = a_ll(p);
64 while (!a_sc(p, old & v));
65 a_post_llsc();
66 return old;
67 }
68 #endif
69
70 #ifndef a_fetch_or
71 #define a_fetch_or a_fetch_or
a_fetch_or(volatile int * p,int v)72 static inline int a_fetch_or(volatile int *p, int v)
73 {
74 int old;
75 a_pre_llsc();
76 do old = a_ll(p);
77 while (!a_sc(p, old | v));
78 a_post_llsc();
79 return old;
80 }
81 #endif
82
83 #endif
84
85 #ifdef a_ll_p
86
87 #ifndef a_cas_p
88 #define a_cas_p a_cas_p
a_cas_p(volatile void * p,void * t,void * s)89 static inline void *a_cas_p(volatile void *p, void *t, void *s)
90 {
91 void *old;
92 a_pre_llsc();
93 do old = a_ll_p(p);
94 while (old==t && !a_sc_p(p, s));
95 a_post_llsc();
96 return old;
97 }
98 #endif
99
100 #endif
101
102 #ifndef a_cas
103 #error missing definition of a_cas
104 #endif
105
106 #ifndef a_swap
107 #define a_swap a_swap
a_swap(volatile int * p,int v)108 static inline int a_swap(volatile int *p, int v)
109 {
110 int old;
111 do old = *p;
112 while (a_cas(p, old, v) != old);
113 return old;
114 }
115 #endif
116
117 #ifndef a_fetch_add
118 #define a_fetch_add a_fetch_add
a_fetch_add(volatile int * p,int v)119 static inline int a_fetch_add(volatile int *p, int v)
120 {
121 int old;
122 do old = *p;
123 while (a_cas(p, old, (unsigned)old+v) != old);
124 return old;
125 }
126 #endif
127
128 #ifndef a_fetch_and
129 #define a_fetch_and a_fetch_and
a_fetch_and(volatile int * p,int v)130 static inline int a_fetch_and(volatile int *p, int v)
131 {
132 int old;
133 do old = *p;
134 while (a_cas(p, old, old&v) != old);
135 return old;
136 }
137 #endif
138 #ifndef a_fetch_or
139 #define a_fetch_or a_fetch_or
a_fetch_or(volatile int * p,int v)140 static inline int a_fetch_or(volatile int *p, int v)
141 {
142 int old;
143 do old = *p;
144 while (a_cas(p, old, old|v) != old);
145 return old;
146 }
147 #endif
148
149 #ifndef a_and
150 #define a_and a_and
a_and(volatile int * p,int v)151 static inline void a_and(volatile int *p, int v)
152 {
153 a_fetch_and(p, v);
154 }
155 #endif
156
157 #ifndef a_or
158 #define a_or a_or
a_or(volatile int * p,int v)159 static inline void a_or(volatile int *p, int v)
160 {
161 a_fetch_or(p, v);
162 }
163 #endif
164
165 #ifndef a_inc
166 #define a_inc a_inc
a_inc(volatile int * p)167 static inline void a_inc(volatile int *p)
168 {
169 a_fetch_add(p, 1);
170 }
171 #endif
172
173 #ifndef a_dec
174 #define a_dec a_dec
a_dec(volatile int * p)175 static inline void a_dec(volatile int *p)
176 {
177 a_fetch_add(p, -1);
178 }
179 #endif
180
181 #ifndef a_store
182 #define a_store a_store
183 #ifdef ENABLE_HWASAN
184 __attribute__((no_sanitize("hwaddress")))
185 #endif
a_store(volatile int * p,int v)186 static inline void a_store(volatile int *p, int v)
187 {
188 #ifdef a_barrier
189 a_barrier();
190 *p = v;
191 a_barrier();
192 #else
193 a_swap(p, v);
194 #endif
195 }
196 #endif
197
198 #ifndef a_barrier
199 #define a_barrier a_barrier
a_barrier()200 static void a_barrier()
201 {
202 volatile int tmp = 0;
203 a_cas(&tmp, 0, 0);
204 }
205 #endif
206
207 #ifndef a_spin
208 #define a_spin a_barrier
209 #endif
210
211 #ifndef a_and_64
212 #define a_and_64 a_and_64
a_and_64(volatile uint64_t * p,uint64_t v)213 static inline void a_and_64(volatile uint64_t *p, uint64_t v)
214 {
215 union { uint64_t v; uint32_t r[2]; } u = { v };
216 if (u.r[0]+1) a_and((int *)p, u.r[0]);
217 if (u.r[1]+1) a_and((int *)p+1, u.r[1]);
218 }
219 #endif
220
221 #ifndef a_or_64
222 #define a_or_64 a_or_64
a_or_64(volatile uint64_t * p,uint64_t v)223 static inline void a_or_64(volatile uint64_t *p, uint64_t v)
224 {
225 union { uint64_t v; uint32_t r[2]; } u = { v };
226 if (u.r[0]) a_or((int *)p, u.r[0]);
227 if (u.r[1]) a_or((int *)p+1, u.r[1]);
228 }
229 #endif
230
231 #ifndef a_cas_p
232 typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1];
233 #define a_cas_p a_cas_p
a_cas_p(volatile void * p,void * t,void * s)234 static inline void *a_cas_p(volatile void *p, void *t, void *s)
235 {
236 return (void *)a_cas((volatile int *)p, (int)t, (int)s);
237 }
238 #endif
239
240 #ifndef a_or_l
241 #define a_or_l a_or_l
a_or_l(volatile void * p,long v)242 static inline void a_or_l(volatile void *p, long v)
243 {
244 if (sizeof(long) == sizeof(int)) a_or(p, v);
245 else a_or_64(p, v);
246 }
247 #endif
248
249 #ifndef a_crash
250 #define a_crash a_crash
a_crash()251 static inline void a_crash()
252 {
253 *(volatile char *)0=0;
254 }
255 #endif
256
257 #ifndef a_ctz_32
258 #define a_ctz_32 a_ctz_32
a_ctz_32(uint32_t x)259 static inline int a_ctz_32(uint32_t x)
260 {
261 #ifdef a_clz_32
262 return 31-a_clz_32(x&-x);
263 #else
264 static const char debruijn32[32] = {
265 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
266 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
267 };
268 return debruijn32[(x&-x)*0x076be629 >> 27];
269 #endif
270 }
271 #endif
272
273 #ifndef a_ctz_64
274 #define a_ctz_64 a_ctz_64
a_ctz_64(uint64_t x)275 static inline int a_ctz_64(uint64_t x)
276 {
277 static const char debruijn64[64] = {
278 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
279 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
280 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
281 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
282 };
283 if (sizeof(long) < 8) {
284 uint32_t y = x;
285 if (!y) {
286 y = x>>32;
287 return 32 + a_ctz_32(y);
288 }
289 return a_ctz_32(y);
290 }
291 return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
292 }
293 #endif
294
a_ctz_l(unsigned long x)295 static inline int a_ctz_l(unsigned long x)
296 {
297 return (sizeof(long) < 8) ? a_ctz_32(x) : a_ctz_64(x);
298 }
299
300 #ifndef a_clz_64
301 #define a_clz_64 a_clz_64
a_clz_64(uint64_t x)302 static inline int a_clz_64(uint64_t x)
303 {
304 #ifdef a_clz_32
305 if (x>>32)
306 return a_clz_32(x>>32);
307 return a_clz_32(x) + 32;
308 #else
309 uint32_t y;
310 int r;
311 if (x>>32) y=x>>32, r=0; else y=x, r=32;
312 if (y>>16) y>>=16; else r |= 16;
313 if (y>>8) y>>=8; else r |= 8;
314 if (y>>4) y>>=4; else r |= 4;
315 if (y>>2) y>>=2; else r |= 2;
316 return r | !(y>>1);
317 #endif
318 }
319 #endif
320
321 #ifndef a_clz_32
322 #define a_clz_32 a_clz_32
a_clz_32(uint32_t x)323 static inline int a_clz_32(uint32_t x)
324 {
325 x >>= 1;
326 x |= x >> 1;
327 x |= x >> 2;
328 x |= x >> 4;
329 x |= x >> 8;
330 x |= x >> 16;
331 x++;
332 return 31-a_ctz_32(x);
333 }
334 #endif
335
336 #endif
337