1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
11 // Not intended for direct inclusion. Include sanitizer_atomic.h.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef SANITIZER_ATOMIC_MSVC_H
16 #define SANITIZER_ATOMIC_MSVC_H
17
18 extern "C" void _ReadWriteBarrier();
19 #pragma intrinsic(_ReadWriteBarrier)
20 extern "C" void _mm_mfence();
21 #pragma intrinsic(_mm_mfence)
22 extern "C" void _mm_pause();
23 #pragma intrinsic(_mm_pause)
24 extern "C" char _InterlockedExchange8( // NOLINT
25 char volatile *Addend, char Value); // NOLINT
26 #pragma intrinsic(_InterlockedExchange8)
27 extern "C" short _InterlockedExchange16( // NOLINT
28 short volatile *Addend, short Value); // NOLINT
29 #pragma intrinsic(_InterlockedExchange16)
30 extern "C" long _InterlockedExchange( // NOLINT
31 long volatile *Addend, long Value); // NOLINT
32 #pragma intrinsic(_InterlockedExchange)
33 extern "C" long _InterlockedExchangeAdd( // NOLINT
34 long volatile * Addend, long Value); // NOLINT
35 #pragma intrinsic(_InterlockedExchangeAdd)
36 extern "C" char _InterlockedCompareExchange8( // NOLINT
37 char volatile *Destination, // NOLINT
38 char Exchange, char Comparand); // NOLINT
39 #pragma intrinsic(_InterlockedCompareExchange8)
40 extern "C" short _InterlockedCompareExchange16( // NOLINT
41 short volatile *Destination, // NOLINT
42 short Exchange, short Comparand); // NOLINT
43 #pragma intrinsic(_InterlockedCompareExchange16)
44 extern "C"
45 long long _InterlockedCompareExchange64( // NOLINT
46 long long volatile *Destination, // NOLINT
47 long long Exchange, long long Comparand); // NOLINT
48 #pragma intrinsic(_InterlockedCompareExchange64)
49 extern "C" void *_InterlockedCompareExchangePointer(
50 void *volatile *Destination,
51 void *Exchange, void *Comparand);
52 #pragma intrinsic(_InterlockedCompareExchangePointer)
53 extern "C"
54 long __cdecl _InterlockedCompareExchange( // NOLINT
55 long volatile *Destination, // NOLINT
56 long Exchange, long Comparand); // NOLINT
57 #pragma intrinsic(_InterlockedCompareExchange)
58
59 #ifdef _WIN64
60 extern "C" long long _InterlockedExchangeAdd64( // NOLINT
61 long long volatile * Addend, long long Value); // NOLINT
62 #pragma intrinsic(_InterlockedExchangeAdd64)
63 #endif
64
65 namespace __sanitizer {
66
atomic_signal_fence(memory_order)67 INLINE void atomic_signal_fence(memory_order) {
68 _ReadWriteBarrier();
69 }
70
atomic_thread_fence(memory_order)71 INLINE void atomic_thread_fence(memory_order) {
72 _mm_mfence();
73 }
74
proc_yield(int cnt)75 INLINE void proc_yield(int cnt) {
76 for (int i = 0; i < cnt; i++)
77 _mm_pause();
78 }
79
80 template<typename T>
atomic_load(const volatile T * a,memory_order mo)81 INLINE typename T::Type atomic_load(
82 const volatile T *a, memory_order mo) {
83 DCHECK(mo & (memory_order_relaxed | memory_order_consume
84 | memory_order_acquire | memory_order_seq_cst));
85 DCHECK(!((uptr)a % sizeof(*a)));
86 typename T::Type v;
87 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
88 if (mo == memory_order_relaxed) {
89 v = a->val_dont_use;
90 } else {
91 atomic_signal_fence(memory_order_seq_cst);
92 v = a->val_dont_use;
93 atomic_signal_fence(memory_order_seq_cst);
94 }
95 return v;
96 }
97
98 template<typename T>
atomic_store(volatile T * a,typename T::Type v,memory_order mo)99 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
100 DCHECK(mo & (memory_order_relaxed | memory_order_release
101 | memory_order_seq_cst));
102 DCHECK(!((uptr)a % sizeof(*a)));
103 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
104 if (mo == memory_order_relaxed) {
105 a->val_dont_use = v;
106 } else {
107 atomic_signal_fence(memory_order_seq_cst);
108 a->val_dont_use = v;
109 atomic_signal_fence(memory_order_seq_cst);
110 }
111 if (mo == memory_order_seq_cst)
112 atomic_thread_fence(memory_order_seq_cst);
113 }
114
atomic_fetch_add(volatile atomic_uint32_t * a,u32 v,memory_order mo)115 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
116 u32 v, memory_order mo) {
117 (void)mo;
118 DCHECK(!((uptr)a % sizeof(*a)));
119 return (u32)_InterlockedExchangeAdd(
120 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
121 }
122
atomic_fetch_add(volatile atomic_uintptr_t * a,uptr v,memory_order mo)123 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
124 uptr v, memory_order mo) {
125 (void)mo;
126 DCHECK(!((uptr)a % sizeof(*a)));
127 #ifdef _WIN64
128 return (uptr)_InterlockedExchangeAdd64(
129 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT
130 #else
131 return (uptr)_InterlockedExchangeAdd(
132 (volatile long*)&a->val_dont_use, (long)v); // NOLINT
133 #endif
134 }
135
atomic_fetch_sub(volatile atomic_uint32_t * a,u32 v,memory_order mo)136 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
137 u32 v, memory_order mo) {
138 (void)mo;
139 DCHECK(!((uptr)a % sizeof(*a)));
140 return (u32)_InterlockedExchangeAdd(
141 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
142 }
143
atomic_fetch_sub(volatile atomic_uintptr_t * a,uptr v,memory_order mo)144 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
145 uptr v, memory_order mo) {
146 (void)mo;
147 DCHECK(!((uptr)a % sizeof(*a)));
148 #ifdef _WIN64
149 return (uptr)_InterlockedExchangeAdd64(
150 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT
151 #else
152 return (uptr)_InterlockedExchangeAdd(
153 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT
154 #endif
155 }
156
atomic_exchange(volatile atomic_uint8_t * a,u8 v,memory_order mo)157 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
158 u8 v, memory_order mo) {
159 (void)mo;
160 DCHECK(!((uptr)a % sizeof(*a)));
161 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
162 }
163
atomic_exchange(volatile atomic_uint16_t * a,u16 v,memory_order mo)164 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
165 u16 v, memory_order mo) {
166 (void)mo;
167 DCHECK(!((uptr)a % sizeof(*a)));
168 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
169 }
170
atomic_exchange(volatile atomic_uint32_t * a,u32 v,memory_order mo)171 INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
172 u32 v, memory_order mo) {
173 (void)mo;
174 DCHECK(!((uptr)a % sizeof(*a)));
175 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
176 }
177
atomic_compare_exchange_strong(volatile atomic_uint8_t * a,u8 * cmp,u8 xchgv,memory_order mo)178 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
179 u8 *cmp,
180 u8 xchgv,
181 memory_order mo) {
182 (void)mo;
183 DCHECK(!((uptr)a % sizeof(*a)));
184 u8 cmpv = *cmp;
185 #ifdef _WIN64
186 u8 prev = (u8)_InterlockedCompareExchange8(
187 (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
188 #else
189 u8 prev;
190 __asm {
191 mov al, cmpv
192 mov ecx, a
193 mov dl, xchgv
194 lock cmpxchg [ecx], dl
195 mov prev, al
196 }
197 #endif
198 if (prev == cmpv)
199 return true;
200 *cmp = prev;
201 return false;
202 }
203
atomic_compare_exchange_strong(volatile atomic_uintptr_t * a,uptr * cmp,uptr xchg,memory_order mo)204 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
205 uptr *cmp,
206 uptr xchg,
207 memory_order mo) {
208 uptr cmpv = *cmp;
209 uptr prev = (uptr)_InterlockedCompareExchangePointer(
210 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
211 if (prev == cmpv)
212 return true;
213 *cmp = prev;
214 return false;
215 }
216
atomic_compare_exchange_strong(volatile atomic_uint16_t * a,u16 * cmp,u16 xchg,memory_order mo)217 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
218 u16 *cmp,
219 u16 xchg,
220 memory_order mo) {
221 u16 cmpv = *cmp;
222 u16 prev = (u16)_InterlockedCompareExchange16(
223 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
224 if (prev == cmpv)
225 return true;
226 *cmp = prev;
227 return false;
228 }
229
atomic_compare_exchange_strong(volatile atomic_uint32_t * a,u32 * cmp,u32 xchg,memory_order mo)230 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
231 u32 *cmp,
232 u32 xchg,
233 memory_order mo) {
234 u32 cmpv = *cmp;
235 u32 prev = (u32)_InterlockedCompareExchange(
236 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
237 if (prev == cmpv)
238 return true;
239 *cmp = prev;
240 return false;
241 }
242
atomic_compare_exchange_strong(volatile atomic_uint64_t * a,u64 * cmp,u64 xchg,memory_order mo)243 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
244 u64 *cmp,
245 u64 xchg,
246 memory_order mo) {
247 u64 cmpv = *cmp;
248 u64 prev = (u64)_InterlockedCompareExchange64(
249 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
250 if (prev == cmpv)
251 return true;
252 *cmp = prev;
253 return false;
254 }
255
256 template<typename T>
atomic_compare_exchange_weak(volatile T * a,typename T::Type * cmp,typename T::Type xchg,memory_order mo)257 INLINE bool atomic_compare_exchange_weak(volatile T *a,
258 typename T::Type *cmp,
259 typename T::Type xchg,
260 memory_order mo) {
261 return atomic_compare_exchange_strong(a, cmp, xchg, mo);
262 }
263
264 } // namespace __sanitizer
265
266 #endif // SANITIZER_ATOMIC_CLANG_H
267