1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
10 #define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
11
12 #include <__atomic/is_always_lock_free.h>
13 #include <__atomic/memory_order.h>
14 #include <__config>
15 #include <__memory/addressof.h>
16 #include <__type_traits/conditional.h>
17 #include <__type_traits/is_assignable.h>
18 #include <__type_traits/is_trivially_copyable.h>
19 #include <__type_traits/remove_const.h>
20 #include <cstddef>
21 #include <cstring>
22
23 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
24 # pragma GCC system_header
25 #endif
26
27 _LIBCPP_BEGIN_NAMESPACE_STD
28
29 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \
30 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS)
31
32 // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
33 // the default operator= in an object is not volatile, a byte-by-byte copy
34 // is required.
35 template <typename _Tp, typename _Tv> _LIBCPP_HIDE_FROM_ABI
36 typename enable_if<is_assignable<_Tp&, _Tv>::value>::type
__cxx_atomic_assign_volatile(_Tp & __a_value,_Tv const & __val)37 __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
38 __a_value = __val;
39 }
40 template <typename _Tp, typename _Tv> _LIBCPP_HIDE_FROM_ABI
41 typename enable_if<is_assignable<_Tp&, _Tv>::value>::type
__cxx_atomic_assign_volatile(_Tp volatile & __a_value,_Tv volatile const & __val)42 __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
43 volatile char* __to = reinterpret_cast<volatile char*>(std::addressof(__a_value));
44 volatile char* __end = __to + sizeof(_Tp);
45 volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val));
46 while (__to != __end)
47 *__to++ = *__from++;
48 }
49
50 #endif
51
52 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)
53
54 template <typename _Tp>
55 struct __cxx_atomic_base_impl {
56
57 _LIBCPP_HIDE_FROM_ABI
58 #ifndef _LIBCPP_CXX03_LANG
59 __cxx_atomic_base_impl() _NOEXCEPT = default;
60 #else
61 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
62 #endif // _LIBCPP_CXX03_LANG
__cxx_atomic_base_impl__cxx_atomic_base_impl63 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT
64 : __a_value(value) {}
65 _Tp __a_value;
66 };
67
__to_gcc_order(memory_order __order)68 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
69 // Avoid switch statement to make this a constexpr.
70 return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
71 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
72 (__order == memory_order_release ? __ATOMIC_RELEASE:
73 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
74 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL:
75 __ATOMIC_CONSUME))));
76 }
77
__to_gcc_failure_order(memory_order __order)78 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
79 // Avoid switch statement to make this a constexpr.
80 return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
81 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
82 (__order == memory_order_release ? __ATOMIC_RELAXED:
83 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
84 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE:
85 __ATOMIC_CONSUME))));
86 }
87
88 template <typename _Tp>
89 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp> * __a,_Tp __val)90 void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
91 __cxx_atomic_assign_volatile(__a->__a_value, __val);
92 }
93
94 template <typename _Tp>
95 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a,_Tp __val)96 void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
97 __a->__a_value = __val;
98 }
99
100 _LIBCPP_HIDE_FROM_ABI inline
__cxx_atomic_thread_fence(memory_order __order)101 void __cxx_atomic_thread_fence(memory_order __order) {
102 __atomic_thread_fence(__to_gcc_order(__order));
103 }
104
105 _LIBCPP_HIDE_FROM_ABI inline
__cxx_atomic_signal_fence(memory_order __order)106 void __cxx_atomic_signal_fence(memory_order __order) {
107 __atomic_signal_fence(__to_gcc_order(__order));
108 }
109
110 template <typename _Tp>
111 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp> * __a,_Tp __val,memory_order __order)112 void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val,
113 memory_order __order) {
114 __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
115 }
116
117 template <typename _Tp>
118 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a,_Tp __val,memory_order __order)119 void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val,
120 memory_order __order) {
121 __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
122 }
123
124 template <typename _Tp>
125 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp> * __a,memory_order __order)126 _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
127 memory_order __order) {
128 _Tp __ret;
129 __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
130 return __ret;
131 }
132
133 template <typename _Tp>
134 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_load(const __cxx_atomic_base_impl<_Tp> * __a,memory_order __order)135 _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
136 _Tp __ret;
137 __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
138 return __ret;
139 }
140
141 template <typename _Tp>
142 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp> * __a,_Tp __value,memory_order __order)143 _Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a,
144 _Tp __value, memory_order __order) {
145 _Tp __ret;
146 __atomic_exchange(
147 std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
148 return __ret;
149 }
150
151 template <typename _Tp>
152 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a,_Tp __value,memory_order __order)153 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value,
154 memory_order __order) {
155 _Tp __ret;
156 __atomic_exchange(
157 std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
158 return __ret;
159 }
160
161 template <typename _Tp>
162 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_base_impl<_Tp> * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)163 bool __cxx_atomic_compare_exchange_strong(
164 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
165 memory_order __success, memory_order __failure) {
166 return __atomic_compare_exchange(
167 std::addressof(__a->__a_value),
168 __expected,
169 std::addressof(__value),
170 false,
171 __to_gcc_order(__success),
172 __to_gcc_failure_order(__failure));
173 }
174
175 template <typename _Tp>
176 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)177 bool __cxx_atomic_compare_exchange_strong(
178 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
179 memory_order __failure) {
180 return __atomic_compare_exchange(
181 std::addressof(__a->__a_value),
182 __expected,
183 std::addressof(__value),
184 false,
185 __to_gcc_order(__success),
186 __to_gcc_failure_order(__failure));
187 }
188
189 template <typename _Tp>
190 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_base_impl<_Tp> * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)191 bool __cxx_atomic_compare_exchange_weak(
192 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
193 memory_order __success, memory_order __failure) {
194 return __atomic_compare_exchange(
195 std::addressof(__a->__a_value),
196 __expected,
197 std::addressof(__value),
198 true,
199 __to_gcc_order(__success),
200 __to_gcc_failure_order(__failure));
201 }
202
203 template <typename _Tp>
204 _LIBCPP_HIDE_FROM_ABI
__cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a,_Tp * __expected,_Tp __value,memory_order __success,memory_order __failure)205 bool __cxx_atomic_compare_exchange_weak(
206 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
207 memory_order __failure) {
208 return __atomic_compare_exchange(
209 std::addressof(__a->__a_value),
210 __expected,
211 std::addressof(__value),
212 true,
213 __to_gcc_order(__success),
214 __to_gcc_failure_order(__failure));
215 }
216
217 template <typename _Tp>
218 struct __skip_amt { enum {value = 1}; };
219
220 template <typename _Tp>
221 struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; };
222
223 // FIXME: Haven't figured out what the spec says about using arrays with
224 // atomic_fetch_add. Force a failure rather than creating bad behavior.
225 template <typename _Tp>
226 struct __skip_amt<_Tp[]> { };
227 template <typename _Tp, int n>
228 struct __skip_amt<_Tp[n]> { };
229
230 template <typename _Tp, typename _Td>
231 _LIBCPP_HIDE_FROM_ABI
232 _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a,
233 _Td __delta, memory_order __order) {
234 return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
235 }
236
237 template <typename _Tp, typename _Td>
238 _LIBCPP_HIDE_FROM_ABI
239 _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
240 memory_order __order) {
241 return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
242 }
243
244 template <typename _Tp, typename _Td>
245 _LIBCPP_HIDE_FROM_ABI
246 _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a,
247 _Td __delta, memory_order __order) {
248 return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
249 }
250
251 template <typename _Tp, typename _Td>
252 _LIBCPP_HIDE_FROM_ABI
253 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
254 memory_order __order) {
255 return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
256 }
257
258 template <typename _Tp>
259 _LIBCPP_HIDE_FROM_ABI
260 _Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a,
261 _Tp __pattern, memory_order __order) {
262 return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
263 }
264
265 template <typename _Tp>
266 _LIBCPP_HIDE_FROM_ABI
267 _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a,
268 _Tp __pattern, memory_order __order) {
269 return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
270 }
271
272 template <typename _Tp>
273 _LIBCPP_HIDE_FROM_ABI
274 _Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a,
275 _Tp __pattern, memory_order __order) {
276 return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
277 }
278
279 template <typename _Tp>
280 _LIBCPP_HIDE_FROM_ABI
281 _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
282 memory_order __order) {
283 return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
284 }
285
286 template <typename _Tp>
287 _LIBCPP_HIDE_FROM_ABI
288 _Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a,
289 _Tp __pattern, memory_order __order) {
290 return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
291 }
292
293 template <typename _Tp>
294 _LIBCPP_HIDE_FROM_ABI
295 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
296 memory_order __order) {
297 return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
298 }
299
300 #define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
301
302 #elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
303
304 template <typename _Tp>
305 struct __cxx_atomic_base_impl {
306
307 _LIBCPP_HIDE_FROM_ABI
308 #ifndef _LIBCPP_CXX03_LANG
309 __cxx_atomic_base_impl() _NOEXCEPT = default;
310 #else
311 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
312 #endif // _LIBCPP_CXX03_LANG
313 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT
314 : __a_value(__value) {}
315 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
316 };
317
318 #define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
319
320 _LIBCPP_HIDE_FROM_ABI inline
321 void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
322 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
323 }
324
325 _LIBCPP_HIDE_FROM_ABI inline
326 void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
327 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
328 }
329
330 template<class _Tp>
331 _LIBCPP_HIDE_FROM_ABI
332 void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
333 __c11_atomic_init(std::addressof(__a->__a_value), __val);
334 }
335 template<class _Tp>
336 _LIBCPP_HIDE_FROM_ABI
337 void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT {
338 __c11_atomic_init(std::addressof(__a->__a_value), __val);
339 }
340
341 template<class _Tp>
342 _LIBCPP_HIDE_FROM_ABI
343 void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
344 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
345 }
346 template<class _Tp>
347 _LIBCPP_HIDE_FROM_ABI
348 void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT {
349 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
350 }
351
352 template<class _Tp>
353 _LIBCPP_HIDE_FROM_ABI
354 _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
355 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
356 return __c11_atomic_load(
357 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
358 }
359 template<class _Tp>
360 _LIBCPP_HIDE_FROM_ABI
361 _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
362 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
363 return __c11_atomic_load(
364 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
365 }
366
367 template<class _Tp>
368 _LIBCPP_HIDE_FROM_ABI
369 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
370 return __c11_atomic_exchange(
371 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
372 }
373 template<class _Tp>
374 _LIBCPP_HIDE_FROM_ABI
375 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT {
376 return __c11_atomic_exchange(
377 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
378 }
379
380 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
381 // Avoid switch statement to make this a constexpr.
382 return __order == memory_order_release ? memory_order_relaxed:
383 (__order == memory_order_acq_rel ? memory_order_acquire:
384 __order);
385 }
386
387 template<class _Tp>
388 _LIBCPP_HIDE_FROM_ABI
389 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
390 return __c11_atomic_compare_exchange_strong(
391 std::addressof(__a->__a_value),
392 __expected,
393 __value,
394 static_cast<__memory_order_underlying_t>(__success),
395 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
396 }
397 template<class _Tp>
398 _LIBCPP_HIDE_FROM_ABI
399 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
400 return __c11_atomic_compare_exchange_strong(
401 std::addressof(__a->__a_value),
402 __expected,
403 __value,
404 static_cast<__memory_order_underlying_t>(__success),
405 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
406 }
407
408 template<class _Tp>
409 _LIBCPP_HIDE_FROM_ABI
410 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
411 return __c11_atomic_compare_exchange_weak(
412 std::addressof(__a->__a_value),
413 __expected,
414 __value,
415 static_cast<__memory_order_underlying_t>(__success),
416 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
417 }
418 template<class _Tp>
419 _LIBCPP_HIDE_FROM_ABI
420 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
421 return __c11_atomic_compare_exchange_weak(
422 std::addressof(__a->__a_value),
423 __expected,
424 __value,
425 static_cast<__memory_order_underlying_t>(__success),
426 static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
427 }
428
429 template<class _Tp>
430 _LIBCPP_HIDE_FROM_ABI
431 _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
432 return __c11_atomic_fetch_add(
433 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
434 }
435 template<class _Tp>
436 _LIBCPP_HIDE_FROM_ABI
437 _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
438 return __c11_atomic_fetch_add(
439 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
440 }
441
442 template<class _Tp>
443 _LIBCPP_HIDE_FROM_ABI
444 _Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
445 return __c11_atomic_fetch_add(
446 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
447 }
448 template<class _Tp>
449 _LIBCPP_HIDE_FROM_ABI
450 _Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
451 return __c11_atomic_fetch_add(
452 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
453 }
454
455 template<class _Tp>
456 _LIBCPP_HIDE_FROM_ABI
457 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
458 return __c11_atomic_fetch_sub(
459 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
460 }
461 template<class _Tp>
462 _LIBCPP_HIDE_FROM_ABI
463 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
464 return __c11_atomic_fetch_sub(
465 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
466 }
467 template<class _Tp>
468 _LIBCPP_HIDE_FROM_ABI
469 _Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
470 return __c11_atomic_fetch_sub(
471 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
472 }
473 template<class _Tp>
474 _LIBCPP_HIDE_FROM_ABI
475 _Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
476 return __c11_atomic_fetch_sub(
477 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
478 }
479
480 template<class _Tp>
481 _LIBCPP_HIDE_FROM_ABI
482 _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
483 return __c11_atomic_fetch_and(
484 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
485 }
486 template<class _Tp>
487 _LIBCPP_HIDE_FROM_ABI
488 _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
489 return __c11_atomic_fetch_and(
490 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
491 }
492
493 template<class _Tp>
494 _LIBCPP_HIDE_FROM_ABI
495 _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
496 return __c11_atomic_fetch_or(
497 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
498 }
499 template<class _Tp>
500 _LIBCPP_HIDE_FROM_ABI
501 _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
502 return __c11_atomic_fetch_or(
503 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
504 }
505
506 template<class _Tp>
507 _LIBCPP_HIDE_FROM_ABI
508 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
509 return __c11_atomic_fetch_xor(
510 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
511 }
512 template<class _Tp>
513 _LIBCPP_HIDE_FROM_ABI
514 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
515 return __c11_atomic_fetch_xor(
516 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
517 }
518
519 #endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
520
521 #ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
522
523 template<typename _Tp>
524 struct __cxx_atomic_lock_impl {
525
526 _LIBCPP_HIDE_FROM_ABI
527 __cxx_atomic_lock_impl() _NOEXCEPT
528 : __a_value(), __a_lock(0) {}
529 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit
530 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT
531 : __a_value(value), __a_lock(0) {}
532
533 _Tp __a_value;
534 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock;
535
536 _LIBCPP_HIDE_FROM_ABI void __lock() const volatile {
537 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
538 /*spin*/;
539 }
540 _LIBCPP_HIDE_FROM_ABI void __lock() const {
541 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
542 /*spin*/;
543 }
544 _LIBCPP_HIDE_FROM_ABI void __unlock() const volatile {
545 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
546 }
547 _LIBCPP_HIDE_FROM_ABI void __unlock() const {
548 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
549 }
550 _LIBCPP_HIDE_FROM_ABI _Tp __read() const volatile {
551 __lock();
552 _Tp __old;
553 __cxx_atomic_assign_volatile(__old, __a_value);
554 __unlock();
555 return __old;
556 }
557 _LIBCPP_HIDE_FROM_ABI _Tp __read() const {
558 __lock();
559 _Tp __old = __a_value;
560 __unlock();
561 return __old;
562 }
563 };
564
565 template <typename _Tp>
566 _LIBCPP_HIDE_FROM_ABI
567 void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) {
568 __cxx_atomic_assign_volatile(__a->__a_value, __val);
569 }
570 template <typename _Tp>
571 _LIBCPP_HIDE_FROM_ABI
572 void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) {
573 __a->__a_value = __val;
574 }
575
576 template <typename _Tp>
577 _LIBCPP_HIDE_FROM_ABI
578 void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) {
579 __a->__lock();
580 __cxx_atomic_assign_volatile(__a->__a_value, __val);
581 __a->__unlock();
582 }
583 template <typename _Tp>
584 _LIBCPP_HIDE_FROM_ABI
585 void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) {
586 __a->__lock();
587 __a->__a_value = __val;
588 __a->__unlock();
589 }
590
591 template <typename _Tp>
592 _LIBCPP_HIDE_FROM_ABI
593 _Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
594 return __a->__read();
595 }
596 template <typename _Tp>
597 _LIBCPP_HIDE_FROM_ABI
598 _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
599 return __a->__read();
600 }
601
602 template <typename _Tp>
603 _LIBCPP_HIDE_FROM_ABI
604 _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
605 __a->__lock();
606 _Tp __old;
607 __cxx_atomic_assign_volatile(__old, __a->__a_value);
608 __cxx_atomic_assign_volatile(__a->__a_value, __value);
609 __a->__unlock();
610 return __old;
611 }
612 template <typename _Tp>
613 _LIBCPP_HIDE_FROM_ABI
614 _Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
615 __a->__lock();
616 _Tp __old = __a->__a_value;
617 __a->__a_value = __value;
618 __a->__unlock();
619 return __old;
620 }
621
622 template <typename _Tp>
623 _LIBCPP_HIDE_FROM_ABI
624 bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a,
625 _Tp* __expected, _Tp __value, memory_order, memory_order) {
626 _Tp __temp;
627 __a->__lock();
628 __cxx_atomic_assign_volatile(__temp, __a->__a_value);
629 bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
630 if(__ret)
631 __cxx_atomic_assign_volatile(__a->__a_value, __value);
632 else
633 __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
634 __a->__unlock();
635 return __ret;
636 }
637 template <typename _Tp>
638 _LIBCPP_HIDE_FROM_ABI
639 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a,
640 _Tp* __expected, _Tp __value, memory_order, memory_order) {
641 __a->__lock();
642 bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
643 if(__ret)
644 std::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
645 else
646 std::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
647 __a->__unlock();
648 return __ret;
649 }
650
651 template <typename _Tp>
652 _LIBCPP_HIDE_FROM_ABI
653 bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a,
654 _Tp* __expected, _Tp __value, memory_order, memory_order) {
655 _Tp __temp;
656 __a->__lock();
657 __cxx_atomic_assign_volatile(__temp, __a->__a_value);
658 bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
659 if(__ret)
660 __cxx_atomic_assign_volatile(__a->__a_value, __value);
661 else
662 __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
663 __a->__unlock();
664 return __ret;
665 }
666 template <typename _Tp>
667 _LIBCPP_HIDE_FROM_ABI
668 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a,
669 _Tp* __expected, _Tp __value, memory_order, memory_order) {
670 __a->__lock();
671 bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
672 if(__ret)
673 std::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
674 else
675 std::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
676 __a->__unlock();
677 return __ret;
678 }
679
680 template <typename _Tp, typename _Td>
681 _LIBCPP_HIDE_FROM_ABI
682 _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a,
683 _Td __delta, memory_order) {
684 __a->__lock();
685 _Tp __old;
686 __cxx_atomic_assign_volatile(__old, __a->__a_value);
687 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta));
688 __a->__unlock();
689 return __old;
690 }
691 template <typename _Tp, typename _Td>
692 _LIBCPP_HIDE_FROM_ABI
693 _Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a,
694 _Td __delta, memory_order) {
695 __a->__lock();
696 _Tp __old = __a->__a_value;
697 __a->__a_value += __delta;
698 __a->__unlock();
699 return __old;
700 }
701
702 template <typename _Tp, typename _Td>
703 _LIBCPP_HIDE_FROM_ABI
704 _Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a,
705 ptrdiff_t __delta, memory_order) {
706 __a->__lock();
707 _Tp* __old;
708 __cxx_atomic_assign_volatile(__old, __a->__a_value);
709 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta);
710 __a->__unlock();
711 return __old;
712 }
713 template <typename _Tp, typename _Td>
714 _LIBCPP_HIDE_FROM_ABI
715 _Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a,
716 ptrdiff_t __delta, memory_order) {
717 __a->__lock();
718 _Tp* __old = __a->__a_value;
719 __a->__a_value += __delta;
720 __a->__unlock();
721 return __old;
722 }
723
724 template <typename _Tp, typename _Td>
725 _LIBCPP_HIDE_FROM_ABI
726 _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a,
727 _Td __delta, memory_order) {
728 __a->__lock();
729 _Tp __old;
730 __cxx_atomic_assign_volatile(__old, __a->__a_value);
731 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta));
732 __a->__unlock();
733 return __old;
734 }
735 template <typename _Tp, typename _Td>
736 _LIBCPP_HIDE_FROM_ABI
737 _Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a,
738 _Td __delta, memory_order) {
739 __a->__lock();
740 _Tp __old = __a->__a_value;
741 __a->__a_value -= __delta;
742 __a->__unlock();
743 return __old;
744 }
745
746 template <typename _Tp>
747 _LIBCPP_HIDE_FROM_ABI
748 _Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a,
749 _Tp __pattern, memory_order) {
750 __a->__lock();
751 _Tp __old;
752 __cxx_atomic_assign_volatile(__old, __a->__a_value);
753 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern));
754 __a->__unlock();
755 return __old;
756 }
757 template <typename _Tp>
758 _LIBCPP_HIDE_FROM_ABI
759 _Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a,
760 _Tp __pattern, memory_order) {
761 __a->__lock();
762 _Tp __old = __a->__a_value;
763 __a->__a_value &= __pattern;
764 __a->__unlock();
765 return __old;
766 }
767
768 template <typename _Tp>
769 _LIBCPP_HIDE_FROM_ABI
770 _Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a,
771 _Tp __pattern, memory_order) {
772 __a->__lock();
773 _Tp __old;
774 __cxx_atomic_assign_volatile(__old, __a->__a_value);
775 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern));
776 __a->__unlock();
777 return __old;
778 }
779 template <typename _Tp>
780 _LIBCPP_HIDE_FROM_ABI
781 _Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a,
782 _Tp __pattern, memory_order) {
783 __a->__lock();
784 _Tp __old = __a->__a_value;
785 __a->__a_value |= __pattern;
786 __a->__unlock();
787 return __old;
788 }
789
790 template <typename _Tp>
791 _LIBCPP_HIDE_FROM_ABI
792 _Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a,
793 _Tp __pattern, memory_order) {
794 __a->__lock();
795 _Tp __old;
796 __cxx_atomic_assign_volatile(__old, __a->__a_value);
797 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern));
798 __a->__unlock();
799 return __old;
800 }
801 template <typename _Tp>
802 _LIBCPP_HIDE_FROM_ABI
803 _Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a,
804 _Tp __pattern, memory_order) {
805 __a->__lock();
806 _Tp __old = __a->__a_value;
807 __a->__a_value ^= __pattern;
808 __a->__unlock();
809 return __old;
810 }
811
812 template <typename _Tp,
813 typename _Base = typename conditional<__libcpp_is_always_lock_free<_Tp>::__value,
814 __cxx_atomic_base_impl<_Tp>,
815 __cxx_atomic_lock_impl<_Tp> >::type>
816 #else
817 template <typename _Tp,
818 typename _Base = __cxx_atomic_base_impl<_Tp> >
819 #endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
820 struct __cxx_atomic_impl : public _Base {
821 static_assert(is_trivially_copyable<_Tp>::value,
822 "std::atomic<T> requires that 'T' be a trivially copyable type");
823
824 _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
825 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT
826 : _Base(__value) {}
827 };
828
829 _LIBCPP_END_NAMESPACE_STD
830
831 #endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
832