• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*-
2  * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org>
3  *                    David Chisnall <theraven@FreeBSD.org>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  * $FreeBSD$
28  */
29 
30 #pragma once
31 
32 #include <sys/cdefs.h>
33 #include <sys/types.h>
34 #include <stdbool.h>
35 
36 /*
37  * C: Do it ourselves.
38  * Note that the runtime representation defined here should be compatible
39  * with the C++ one, i.e. an _Atomic(T) needs to contain the same
40  * bits as a T.
41  */
42 
43 #include <stddef.h>  /* For ptrdiff_t. */
44 #include <stdint.h>  /* TODO: don't drag in all the macros, just the types. */
45 // Include uchar.h only when available.  Bionic's stdatomic.h is also used for
46 // the host (via a copy in prebuilts/clang) and uchar.h is not available in the
47 // glibc used for the host.
48 #if defined(__BIONIC__)
49 # include <uchar.h>  /* For char16_t and char32_t.              */
50 #endif
51 
52 /*
53  * 7.17.1 Atomic lock-free macros.
54  */
55 
56 #ifdef __GCC_ATOMIC_BOOL_LOCK_FREE
57 #define	ATOMIC_BOOL_LOCK_FREE		__GCC_ATOMIC_BOOL_LOCK_FREE
58 #endif
59 #ifdef __GCC_ATOMIC_CHAR_LOCK_FREE
60 #define	ATOMIC_CHAR_LOCK_FREE		__GCC_ATOMIC_CHAR_LOCK_FREE
61 #endif
62 #ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE
63 #define	ATOMIC_CHAR16_T_LOCK_FREE	__GCC_ATOMIC_CHAR16_T_LOCK_FREE
64 #endif
65 #ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE
66 #define	ATOMIC_CHAR32_T_LOCK_FREE	__GCC_ATOMIC_CHAR32_T_LOCK_FREE
67 #endif
68 #ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE
69 #define	ATOMIC_WCHAR_T_LOCK_FREE	__GCC_ATOMIC_WCHAR_T_LOCK_FREE
70 #endif
71 #ifdef __GCC_ATOMIC_SHORT_LOCK_FREE
72 #define	ATOMIC_SHORT_LOCK_FREE		__GCC_ATOMIC_SHORT_LOCK_FREE
73 #endif
74 #ifdef __GCC_ATOMIC_INT_LOCK_FREE
75 #define	ATOMIC_INT_LOCK_FREE		__GCC_ATOMIC_INT_LOCK_FREE
76 #endif
77 #ifdef __GCC_ATOMIC_LONG_LOCK_FREE
78 #define	ATOMIC_LONG_LOCK_FREE		__GCC_ATOMIC_LONG_LOCK_FREE
79 #endif
80 #ifdef __GCC_ATOMIC_LLONG_LOCK_FREE
81 #define	ATOMIC_LLONG_LOCK_FREE		__GCC_ATOMIC_LLONG_LOCK_FREE
82 #endif
83 #ifdef __GCC_ATOMIC_POINTER_LOCK_FREE
84 #define	ATOMIC_POINTER_LOCK_FREE	__GCC_ATOMIC_POINTER_LOCK_FREE
85 #endif
86 
87 /*
88  * 7.17.2 Initialization.
89  */
90 
91 #define	ATOMIC_VAR_INIT(value)		(value)
92 #define	atomic_init(obj, value)		__c11_atomic_init(obj, value)
93 
94 /*
95  * Clang and recent GCC both provide predefined macros for the memory
96  * orderings.  If we are using a compiler that doesn't define them, use the
97  * clang values - these will be ignored in the fallback path.
98  */
99 
100 #ifndef __ATOMIC_RELAXED
101 #define __ATOMIC_RELAXED		0
102 #endif
103 #ifndef __ATOMIC_CONSUME
104 #define __ATOMIC_CONSUME		1
105 #endif
106 #ifndef __ATOMIC_ACQUIRE
107 #define __ATOMIC_ACQUIRE		2
108 #endif
109 #ifndef __ATOMIC_RELEASE
110 #define __ATOMIC_RELEASE		3
111 #endif
112 #ifndef __ATOMIC_ACQ_REL
113 #define __ATOMIC_ACQ_REL		4
114 #endif
115 #ifndef __ATOMIC_SEQ_CST
116 #define __ATOMIC_SEQ_CST		5
117 #endif
118 
119 /*
120  * 7.17.3 Order and consistency.
121  *
122  * The memory_order_* constants that denote the barrier behaviour of the
123  * atomic operations.
124  * The enum values must be identical to those used by the
125  * C++ <atomic> header.
126  */
127 
128 typedef enum {
129 	memory_order_relaxed = __ATOMIC_RELAXED,
130 	memory_order_consume = __ATOMIC_CONSUME,
131 	memory_order_acquire = __ATOMIC_ACQUIRE,
132 	memory_order_release = __ATOMIC_RELEASE,
133 	memory_order_acq_rel = __ATOMIC_ACQ_REL,
134 	memory_order_seq_cst = __ATOMIC_SEQ_CST
135 } memory_order;
136 
137 /*
138  * 7.17.4 Fences.
139  */
140 
atomic_thread_fence(memory_order __order)141 static __inline void atomic_thread_fence(memory_order __order __attribute__((unused))) {
142 	__c11_atomic_thread_fence(__order);
143 }
144 
atomic_signal_fence(memory_order __order)145 static __inline void atomic_signal_fence(memory_order __order __attribute__((unused))) {
146 	__c11_atomic_signal_fence(__order);
147 }
148 
149 /*
150  * 7.17.5 Lock-free property.
151  */
152 
153 #define	atomic_is_lock_free(obj) __c11_atomic_is_lock_free(sizeof(*(obj)))
154 
155 /*
156  * 7.17.6 Atomic integer types.
157  */
158 
159 typedef _Atomic(bool)			atomic_bool;
160 typedef _Atomic(char)			atomic_char;
161 typedef _Atomic(signed char)		atomic_schar;
162 typedef _Atomic(unsigned char)		atomic_uchar;
163 typedef _Atomic(short)			atomic_short;
164 typedef _Atomic(unsigned short)		atomic_ushort;
165 typedef _Atomic(int)			atomic_int;
166 typedef _Atomic(unsigned int)		atomic_uint;
167 typedef _Atomic(long)			atomic_long;
168 typedef _Atomic(unsigned long)		atomic_ulong;
169 typedef _Atomic(long long)		atomic_llong;
170 typedef _Atomic(unsigned long long)	atomic_ullong;
171 #if defined(__BIONIC__) || (defined(__cplusplus) && __cplusplus >= 201103L)
172   typedef _Atomic(char16_t)		atomic_char16_t;
173   typedef _Atomic(char32_t)		atomic_char32_t;
174 #endif
175 typedef _Atomic(wchar_t)		atomic_wchar_t;
176 typedef _Atomic(int_least8_t)		atomic_int_least8_t;
177 typedef _Atomic(uint_least8_t)	atomic_uint_least8_t;
178 typedef _Atomic(int_least16_t)	atomic_int_least16_t;
179 typedef _Atomic(uint_least16_t)	atomic_uint_least16_t;
180 typedef _Atomic(int_least32_t)	atomic_int_least32_t;
181 typedef _Atomic(uint_least32_t)	atomic_uint_least32_t;
182 typedef _Atomic(int_least64_t)	atomic_int_least64_t;
183 typedef _Atomic(uint_least64_t)	atomic_uint_least64_t;
184 typedef _Atomic(int_fast8_t)		atomic_int_fast8_t;
185 typedef _Atomic(uint_fast8_t)		atomic_uint_fast8_t;
186 typedef _Atomic(int_fast16_t)		atomic_int_fast16_t;
187 typedef _Atomic(uint_fast16_t)	atomic_uint_fast16_t;
188 typedef _Atomic(int_fast32_t)		atomic_int_fast32_t;
189 typedef _Atomic(uint_fast32_t)	atomic_uint_fast32_t;
190 typedef _Atomic(int_fast64_t)		atomic_int_fast64_t;
191 typedef _Atomic(uint_fast64_t)	atomic_uint_fast64_t;
192 typedef _Atomic(intptr_t)		atomic_intptr_t;
193 typedef _Atomic(uintptr_t)		atomic_uintptr_t;
194 typedef _Atomic(size_t)		atomic_size_t;
195 typedef _Atomic(ptrdiff_t)		atomic_ptrdiff_t;
196 typedef _Atomic(intmax_t)		atomic_intmax_t;
197 typedef _Atomic(uintmax_t)		atomic_uintmax_t;
198 
199 /*
200  * 7.17.7 Operations on atomic types.
201  */
202 
203 /*
204  * Compiler-specific operations.
205  */
206 
207 #define	atomic_compare_exchange_strong_explicit(object, expected,	\
208     desired, success, failure)						\
209 	__c11_atomic_compare_exchange_strong(object, expected, desired,	\
210 	    success, failure)
211 #define	atomic_compare_exchange_weak_explicit(object, expected,		\
212     desired, success, failure)						\
213 	__c11_atomic_compare_exchange_weak(object, expected, desired,	\
214 	    success, failure)
215 #define	atomic_exchange_explicit(object, desired, order)		\
216 	__c11_atomic_exchange(object, desired, order)
217 #define	atomic_fetch_add_explicit(object, operand, order)		\
218 	__c11_atomic_fetch_add(object, operand, order)
219 #define	atomic_fetch_and_explicit(object, operand, order)		\
220 	__c11_atomic_fetch_and(object, operand, order)
221 #define	atomic_fetch_or_explicit(object, operand, order)		\
222 	__c11_atomic_fetch_or(object, operand, order)
223 #define	atomic_fetch_sub_explicit(object, operand, order)		\
224 	__c11_atomic_fetch_sub(object, operand, order)
225 #define	atomic_fetch_xor_explicit(object, operand, order)		\
226 	__c11_atomic_fetch_xor(object, operand, order)
227 #define	atomic_load_explicit(object, order)				\
228 	__c11_atomic_load(object, order)
229 #define	atomic_store_explicit(object, desired, order)			\
230 	__c11_atomic_store(object, desired, order)
231 
232 /*
233  * Convenience functions.
234  */
235 
236 #define	atomic_compare_exchange_strong(object, expected, desired)	\
237 	atomic_compare_exchange_strong_explicit(object, expected,	\
238 	    desired, memory_order_seq_cst, memory_order_seq_cst)
239 #define	atomic_compare_exchange_weak(object, expected, desired)		\
240 	atomic_compare_exchange_weak_explicit(object, expected,		\
241 	    desired, memory_order_seq_cst, memory_order_seq_cst)
242 #define	atomic_exchange(object, desired)				\
243 	atomic_exchange_explicit(object, desired, memory_order_seq_cst)
244 #define	atomic_fetch_add(object, operand)				\
245 	atomic_fetch_add_explicit(object, operand, memory_order_seq_cst)
246 #define	atomic_fetch_and(object, operand)				\
247 	atomic_fetch_and_explicit(object, operand, memory_order_seq_cst)
248 #define	atomic_fetch_or(object, operand)				\
249 	atomic_fetch_or_explicit(object, operand, memory_order_seq_cst)
250 #define	atomic_fetch_sub(object, operand)				\
251 	atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst)
252 #define	atomic_fetch_xor(object, operand)				\
253 	atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst)
254 #define	atomic_load(object)						\
255 	atomic_load_explicit(object, memory_order_seq_cst)
256 #define	atomic_store(object, desired)					\
257 	atomic_store_explicit(object, desired, memory_order_seq_cst)
258 
259 /*
260  * 7.17.8 Atomic flag type and operations.
261  *
262  * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some
263  * kind of compiler built-in type we could use?
264  */
265 
266 typedef struct {
267 	atomic_bool	__flag;
268 } atomic_flag;
269 
270 #define	ATOMIC_FLAG_INIT		{ ATOMIC_VAR_INIT(false) }
271 
atomic_flag_test_and_set_explicit(volatile atomic_flag * __object,memory_order __order)272 static __inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag *__object, memory_order __order) {
273 	return (atomic_exchange_explicit(&__object->__flag, 1, __order));
274 }
275 
atomic_flag_clear_explicit(volatile atomic_flag * __object,memory_order __order)276 static __inline void atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order) {
277 	atomic_store_explicit(&__object->__flag, 0, __order);
278 }
279 
atomic_flag_test_and_set(volatile atomic_flag * __object)280 static __inline bool atomic_flag_test_and_set(volatile atomic_flag *__object) {
281 	return (atomic_flag_test_and_set_explicit(__object, memory_order_seq_cst));
282 }
283 
atomic_flag_clear(volatile atomic_flag * __object)284 static __inline void atomic_flag_clear(volatile atomic_flag *__object) {
285 	atomic_flag_clear_explicit(__object, memory_order_seq_cst);
286 }
287