• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- tsan_interface_atomic.cc ------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
15 // For background see C++11 standard.  A slightly older, publicly
16 // available draft of the standard (not entirely up-to-date, but close enough
17 // for casual browsing) is available here:
18 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
19 // The following page contains more background information:
20 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
21 
22 #include "sanitizer_common/sanitizer_placement_new.h"
23 #include "sanitizer_common/sanitizer_stacktrace.h"
24 #include "sanitizer_common/sanitizer_mutex.h"
25 #include "tsan_flags.h"
26 #include "tsan_interface.h"
27 #include "tsan_rtl.h"
28 
29 using namespace __tsan;  // NOLINT
30 
31 #if !defined(SANITIZER_GO) && __TSAN_HAS_INT128
32 // Protects emulation of 128-bit atomic operations.
33 static StaticSpinMutex mutex128;
34 #endif
35 
IsLoadOrder(morder mo)36 static bool IsLoadOrder(morder mo) {
37   return mo == mo_relaxed || mo == mo_consume
38       || mo == mo_acquire || mo == mo_seq_cst;
39 }
40 
IsStoreOrder(morder mo)41 static bool IsStoreOrder(morder mo) {
42   return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
43 }
44 
IsReleaseOrder(morder mo)45 static bool IsReleaseOrder(morder mo) {
46   return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
47 }
48 
IsAcquireOrder(morder mo)49 static bool IsAcquireOrder(morder mo) {
50   return mo == mo_consume || mo == mo_acquire
51       || mo == mo_acq_rel || mo == mo_seq_cst;
52 }
53 
IsAcqRelOrder(morder mo)54 static bool IsAcqRelOrder(morder mo) {
55   return mo == mo_acq_rel || mo == mo_seq_cst;
56 }
57 
func_xchg(volatile T * v,T op)58 template<typename T> T func_xchg(volatile T *v, T op) {
59   T res = __sync_lock_test_and_set(v, op);
60   // __sync_lock_test_and_set does not contain full barrier.
61   __sync_synchronize();
62   return res;
63 }
64 
func_add(volatile T * v,T op)65 template<typename T> T func_add(volatile T *v, T op) {
66   return __sync_fetch_and_add(v, op);
67 }
68 
func_sub(volatile T * v,T op)69 template<typename T> T func_sub(volatile T *v, T op) {
70   return __sync_fetch_and_sub(v, op);
71 }
72 
func_and(volatile T * v,T op)73 template<typename T> T func_and(volatile T *v, T op) {
74   return __sync_fetch_and_and(v, op);
75 }
76 
func_or(volatile T * v,T op)77 template<typename T> T func_or(volatile T *v, T op) {
78   return __sync_fetch_and_or(v, op);
79 }
80 
func_xor(volatile T * v,T op)81 template<typename T> T func_xor(volatile T *v, T op) {
82   return __sync_fetch_and_xor(v, op);
83 }
84 
func_nand(volatile T * v,T op)85 template<typename T> T func_nand(volatile T *v, T op) {
86   // clang does not support __sync_fetch_and_nand.
87   T cmp = *v;
88   for (;;) {
89     T newv = ~(cmp & op);
90     T cur = __sync_val_compare_and_swap(v, cmp, newv);
91     if (cmp == cur)
92       return cmp;
93     cmp = cur;
94   }
95 }
96 
func_cas(volatile T * v,T cmp,T xch)97 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
98   return __sync_val_compare_and_swap(v, cmp, xch);
99 }
100 
101 // clang does not support 128-bit atomic ops.
102 // Atomic ops are executed under tsan internal mutex,
103 // here we assume that the atomic variables are not accessed
104 // from non-instrumented code.
105 #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(SANITIZER_GO) \
106     && __TSAN_HAS_INT128
func_xchg(volatile a128 * v,a128 op)107 a128 func_xchg(volatile a128 *v, a128 op) {
108   SpinMutexLock lock(&mutex128);
109   a128 cmp = *v;
110   *v = op;
111   return cmp;
112 }
113 
func_add(volatile a128 * v,a128 op)114 a128 func_add(volatile a128 *v, a128 op) {
115   SpinMutexLock lock(&mutex128);
116   a128 cmp = *v;
117   *v = cmp + op;
118   return cmp;
119 }
120 
func_sub(volatile a128 * v,a128 op)121 a128 func_sub(volatile a128 *v, a128 op) {
122   SpinMutexLock lock(&mutex128);
123   a128 cmp = *v;
124   *v = cmp - op;
125   return cmp;
126 }
127 
func_and(volatile a128 * v,a128 op)128 a128 func_and(volatile a128 *v, a128 op) {
129   SpinMutexLock lock(&mutex128);
130   a128 cmp = *v;
131   *v = cmp & op;
132   return cmp;
133 }
134 
func_or(volatile a128 * v,a128 op)135 a128 func_or(volatile a128 *v, a128 op) {
136   SpinMutexLock lock(&mutex128);
137   a128 cmp = *v;
138   *v = cmp | op;
139   return cmp;
140 }
141 
func_xor(volatile a128 * v,a128 op)142 a128 func_xor(volatile a128 *v, a128 op) {
143   SpinMutexLock lock(&mutex128);
144   a128 cmp = *v;
145   *v = cmp ^ op;
146   return cmp;
147 }
148 
func_nand(volatile a128 * v,a128 op)149 a128 func_nand(volatile a128 *v, a128 op) {
150   SpinMutexLock lock(&mutex128);
151   a128 cmp = *v;
152   *v = ~(cmp & op);
153   return cmp;
154 }
155 
func_cas(volatile a128 * v,a128 cmp,a128 xch)156 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
157   SpinMutexLock lock(&mutex128);
158   a128 cur = *v;
159   if (cur == cmp)
160     *v = xch;
161   return cur;
162 }
163 #endif
164 
165 template<typename T>
SizeLog()166 static int SizeLog() {
167   if (sizeof(T) <= 1)
168     return kSizeLog1;
169   else if (sizeof(T) <= 2)
170     return kSizeLog2;
171   else if (sizeof(T) <= 4)
172     return kSizeLog4;
173   else
174     return kSizeLog8;
175   // For 16-byte atomics we also use 8-byte memory access,
176   // this leads to false negatives only in very obscure cases.
177 }
178 
179 #ifndef SANITIZER_GO
to_atomic(const volatile a8 * a)180 static atomic_uint8_t *to_atomic(const volatile a8 *a) {
181   return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
182 }
183 
to_atomic(const volatile a16 * a)184 static atomic_uint16_t *to_atomic(const volatile a16 *a) {
185   return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
186 }
187 #endif
188 
to_atomic(const volatile a32 * a)189 static atomic_uint32_t *to_atomic(const volatile a32 *a) {
190   return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
191 }
192 
to_atomic(const volatile a64 * a)193 static atomic_uint64_t *to_atomic(const volatile a64 *a) {
194   return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
195 }
196 
to_mo(morder mo)197 static memory_order to_mo(morder mo) {
198   switch (mo) {
199   case mo_relaxed: return memory_order_relaxed;
200   case mo_consume: return memory_order_consume;
201   case mo_acquire: return memory_order_acquire;
202   case mo_release: return memory_order_release;
203   case mo_acq_rel: return memory_order_acq_rel;
204   case mo_seq_cst: return memory_order_seq_cst;
205   }
206   CHECK(0);
207   return memory_order_seq_cst;
208 }
209 
210 template<typename T>
NoTsanAtomicLoad(const volatile T * a,morder mo)211 static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
212   return atomic_load(to_atomic(a), to_mo(mo));
213 }
214 
215 #if __TSAN_HAS_INT128 && !defined(SANITIZER_GO)
NoTsanAtomicLoad(const volatile a128 * a,morder mo)216 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
217   SpinMutexLock lock(&mutex128);
218   return *a;
219 }
220 #endif
221 
222 template<typename T>
AtomicLoad(ThreadState * thr,uptr pc,const volatile T * a,morder mo)223 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
224     morder mo) {
225   CHECK(IsLoadOrder(mo));
226   // This fast-path is critical for performance.
227   // Assume the access is atomic.
228   if (!IsAcquireOrder(mo)) {
229     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
230     return NoTsanAtomicLoad(a, mo);
231   }
232   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
233   AcquireImpl(thr, pc, &s->clock);
234   T v = NoTsanAtomicLoad(a, mo);
235   s->mtx.ReadUnlock();
236   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
237   return v;
238 }
239 
240 template<typename T>
NoTsanAtomicStore(volatile T * a,T v,morder mo)241 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
242   atomic_store(to_atomic(a), v, to_mo(mo));
243 }
244 
245 #if __TSAN_HAS_INT128 && !defined(SANITIZER_GO)
NoTsanAtomicStore(volatile a128 * a,a128 v,morder mo)246 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
247   SpinMutexLock lock(&mutex128);
248   *a = v;
249 }
250 #endif
251 
252 template<typename T>
AtomicStore(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)253 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
254     morder mo) {
255   CHECK(IsStoreOrder(mo));
256   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
257   // This fast-path is critical for performance.
258   // Assume the access is atomic.
259   // Strictly saying even relaxed store cuts off release sequence,
260   // so must reset the clock.
261   if (!IsReleaseOrder(mo)) {
262     NoTsanAtomicStore(a, v, mo);
263     return;
264   }
265   __sync_synchronize();
266   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
267   thr->fast_state.IncrementEpoch();
268   // Can't increment epoch w/o writing to the trace as well.
269   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
270   ReleaseImpl(thr, pc, &s->clock);
271   NoTsanAtomicStore(a, v, mo);
272   s->mtx.Unlock();
273 }
274 
275 template<typename T, T (*F)(volatile T *v, T op)>
AtomicRMW(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)276 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
277   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
278   SyncVar *s = 0;
279   if (mo != mo_relaxed) {
280     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
281     thr->fast_state.IncrementEpoch();
282     // Can't increment epoch w/o writing to the trace as well.
283     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
284     if (IsAcqRelOrder(mo))
285       AcquireReleaseImpl(thr, pc, &s->clock);
286     else if (IsReleaseOrder(mo))
287       ReleaseImpl(thr, pc, &s->clock);
288     else if (IsAcquireOrder(mo))
289       AcquireImpl(thr, pc, &s->clock);
290   }
291   v = F(a, v);
292   if (s)
293     s->mtx.Unlock();
294   return v;
295 }
296 
297 template<typename T>
NoTsanAtomicExchange(volatile T * a,T v,morder mo)298 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
299   return func_xchg(a, v);
300 }
301 
302 template<typename T>
NoTsanAtomicFetchAdd(volatile T * a,T v,morder mo)303 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
304   return func_add(a, v);
305 }
306 
307 template<typename T>
NoTsanAtomicFetchSub(volatile T * a,T v,morder mo)308 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
309   return func_sub(a, v);
310 }
311 
312 template<typename T>
NoTsanAtomicFetchAnd(volatile T * a,T v,morder mo)313 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
314   return func_and(a, v);
315 }
316 
317 template<typename T>
NoTsanAtomicFetchOr(volatile T * a,T v,morder mo)318 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
319   return func_or(a, v);
320 }
321 
322 template<typename T>
NoTsanAtomicFetchXor(volatile T * a,T v,morder mo)323 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
324   return func_xor(a, v);
325 }
326 
327 template<typename T>
NoTsanAtomicFetchNand(volatile T * a,T v,morder mo)328 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
329   return func_nand(a, v);
330 }
331 
332 template<typename T>
AtomicExchange(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)333 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
334     morder mo) {
335   return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
336 }
337 
338 template<typename T>
AtomicFetchAdd(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)339 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
340     morder mo) {
341   return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
342 }
343 
344 template<typename T>
AtomicFetchSub(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)345 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
346     morder mo) {
347   return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
348 }
349 
350 template<typename T>
AtomicFetchAnd(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)351 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
352     morder mo) {
353   return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
354 }
355 
356 template<typename T>
AtomicFetchOr(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)357 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
358     morder mo) {
359   return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
360 }
361 
362 template<typename T>
AtomicFetchXor(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)363 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
364     morder mo) {
365   return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
366 }
367 
368 template<typename T>
AtomicFetchNand(ThreadState * thr,uptr pc,volatile T * a,T v,morder mo)369 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
370     morder mo) {
371   return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
372 }
373 
374 template<typename T>
NoTsanAtomicCAS(volatile T * a,T * c,T v,morder mo,morder fmo)375 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
376   return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
377 }
378 
379 #if __TSAN_HAS_INT128
NoTsanAtomicCAS(volatile a128 * a,a128 * c,a128 v,morder mo,morder fmo)380 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
381     morder mo, morder fmo) {
382   a128 old = *c;
383   a128 cur = func_cas(a, old, v);
384   if (cur == old)
385     return true;
386   *c = cur;
387   return false;
388 }
389 #endif
390 
391 template<typename T>
NoTsanAtomicCAS(volatile T * a,T c,T v,morder mo,morder fmo)392 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
393   NoTsanAtomicCAS(a, &c, v, mo, fmo);
394   return c;
395 }
396 
397 template<typename T>
AtomicCAS(ThreadState * thr,uptr pc,volatile T * a,T * c,T v,morder mo,morder fmo)398 static bool AtomicCAS(ThreadState *thr, uptr pc,
399     volatile T *a, T *c, T v, morder mo, morder fmo) {
400   (void)fmo;  // Unused because llvm does not pass it yet.
401   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
402   SyncVar *s = 0;
403   bool write_lock = mo != mo_acquire && mo != mo_consume;
404   if (mo != mo_relaxed) {
405     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
406     thr->fast_state.IncrementEpoch();
407     // Can't increment epoch w/o writing to the trace as well.
408     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
409     if (IsAcqRelOrder(mo))
410       AcquireReleaseImpl(thr, pc, &s->clock);
411     else if (IsReleaseOrder(mo))
412       ReleaseImpl(thr, pc, &s->clock);
413     else if (IsAcquireOrder(mo))
414       AcquireImpl(thr, pc, &s->clock);
415   }
416   T cc = *c;
417   T pr = func_cas(a, cc, v);
418   if (s) {
419     if (write_lock)
420       s->mtx.Unlock();
421     else
422       s->mtx.ReadUnlock();
423   }
424   if (pr == cc)
425     return true;
426   *c = pr;
427   return false;
428 }
429 
430 template<typename T>
AtomicCAS(ThreadState * thr,uptr pc,volatile T * a,T c,T v,morder mo,morder fmo)431 static T AtomicCAS(ThreadState *thr, uptr pc,
432     volatile T *a, T c, T v, morder mo, morder fmo) {
433   AtomicCAS(thr, pc, a, &c, v, mo, fmo);
434   return c;
435 }
436 
437 #ifndef SANITIZER_GO
NoTsanAtomicFence(morder mo)438 static void NoTsanAtomicFence(morder mo) {
439   __sync_synchronize();
440 }
441 
AtomicFence(ThreadState * thr,uptr pc,morder mo)442 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
443   // FIXME(dvyukov): not implemented.
444   __sync_synchronize();
445 }
446 #endif
447 
448 // Interface functions follow.
449 #ifndef SANITIZER_GO
450 
451 // C/C++
452 
453 #define SCOPED_ATOMIC(func, ...) \
454     const uptr callpc = (uptr)__builtin_return_address(0); \
455     uptr pc = StackTrace::GetCurrentPc(); \
456     mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \
457     ThreadState *const thr = cur_thread(); \
458     if (thr->ignore_interceptors) \
459       return NoTsanAtomic##func(__VA_ARGS__); \
460     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
461     ScopedAtomic sa(thr, callpc, a, mo, __func__); \
462     return Atomic##func(thr, pc, __VA_ARGS__); \
463 /**/
464 
465 class ScopedAtomic {
466  public:
ScopedAtomic(ThreadState * thr,uptr pc,const volatile void * a,morder mo,const char * func)467   ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
468                morder mo, const char *func)
469       : thr_(thr) {
470     FuncEntry(thr_, pc);
471     DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
472   }
~ScopedAtomic()473   ~ScopedAtomic() {
474     ProcessPendingSignals(thr_);
475     FuncExit(thr_);
476   }
477  private:
478   ThreadState *thr_;
479 };
480 
AtomicStatInc(ThreadState * thr,uptr size,morder mo,StatType t)481 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
482   StatInc(thr, StatAtomic);
483   StatInc(thr, t);
484   StatInc(thr, size == 1 ? StatAtomic1
485              : size == 2 ? StatAtomic2
486              : size == 4 ? StatAtomic4
487              : size == 8 ? StatAtomic8
488              :             StatAtomic16);
489   StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
490              : mo == mo_consume ? StatAtomicConsume
491              : mo == mo_acquire ? StatAtomicAcquire
492              : mo == mo_release ? StatAtomicRelease
493              : mo == mo_acq_rel ? StatAtomicAcq_Rel
494              :                    StatAtomicSeq_Cst);
495 }
496 
497 extern "C" {
498 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_load(const volatile a8 * a,morder mo)499 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
500   SCOPED_ATOMIC(Load, a, mo);
501 }
502 
503 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_load(const volatile a16 * a,morder mo)504 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
505   SCOPED_ATOMIC(Load, a, mo);
506 }
507 
508 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_load(const volatile a32 * a,morder mo)509 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
510   SCOPED_ATOMIC(Load, a, mo);
511 }
512 
513 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_load(const volatile a64 * a,morder mo)514 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
515   SCOPED_ATOMIC(Load, a, mo);
516 }
517 
518 #if __TSAN_HAS_INT128
519 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_load(const volatile a128 * a,morder mo)520 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
521   SCOPED_ATOMIC(Load, a, mo);
522 }
523 #endif
524 
525 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_store(volatile a8 * a,a8 v,morder mo)526 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
527   SCOPED_ATOMIC(Store, a, v, mo);
528 }
529 
530 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_store(volatile a16 * a,a16 v,morder mo)531 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
532   SCOPED_ATOMIC(Store, a, v, mo);
533 }
534 
535 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_store(volatile a32 * a,a32 v,morder mo)536 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
537   SCOPED_ATOMIC(Store, a, v, mo);
538 }
539 
540 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_store(volatile a64 * a,a64 v,morder mo)541 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
542   SCOPED_ATOMIC(Store, a, v, mo);
543 }
544 
545 #if __TSAN_HAS_INT128
546 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_store(volatile a128 * a,a128 v,morder mo)547 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
548   SCOPED_ATOMIC(Store, a, v, mo);
549 }
550 #endif
551 
552 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_exchange(volatile a8 * a,a8 v,morder mo)553 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
554   SCOPED_ATOMIC(Exchange, a, v, mo);
555 }
556 
557 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_exchange(volatile a16 * a,a16 v,morder mo)558 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
559   SCOPED_ATOMIC(Exchange, a, v, mo);
560 }
561 
562 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_exchange(volatile a32 * a,a32 v,morder mo)563 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
564   SCOPED_ATOMIC(Exchange, a, v, mo);
565 }
566 
567 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_exchange(volatile a64 * a,a64 v,morder mo)568 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
569   SCOPED_ATOMIC(Exchange, a, v, mo);
570 }
571 
572 #if __TSAN_HAS_INT128
573 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_exchange(volatile a128 * a,a128 v,morder mo)574 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
575   SCOPED_ATOMIC(Exchange, a, v, mo);
576 }
577 #endif
578 
579 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_add(volatile a8 * a,a8 v,morder mo)580 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
581   SCOPED_ATOMIC(FetchAdd, a, v, mo);
582 }
583 
584 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_add(volatile a16 * a,a16 v,morder mo)585 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
586   SCOPED_ATOMIC(FetchAdd, a, v, mo);
587 }
588 
589 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_add(volatile a32 * a,a32 v,morder mo)590 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
591   SCOPED_ATOMIC(FetchAdd, a, v, mo);
592 }
593 
594 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_add(volatile a64 * a,a64 v,morder mo)595 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
596   SCOPED_ATOMIC(FetchAdd, a, v, mo);
597 }
598 
599 #if __TSAN_HAS_INT128
600 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_add(volatile a128 * a,a128 v,morder mo)601 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
602   SCOPED_ATOMIC(FetchAdd, a, v, mo);
603 }
604 #endif
605 
606 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_sub(volatile a8 * a,a8 v,morder mo)607 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
608   SCOPED_ATOMIC(FetchSub, a, v, mo);
609 }
610 
611 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_sub(volatile a16 * a,a16 v,morder mo)612 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
613   SCOPED_ATOMIC(FetchSub, a, v, mo);
614 }
615 
616 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_sub(volatile a32 * a,a32 v,morder mo)617 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
618   SCOPED_ATOMIC(FetchSub, a, v, mo);
619 }
620 
621 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_sub(volatile a64 * a,a64 v,morder mo)622 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
623   SCOPED_ATOMIC(FetchSub, a, v, mo);
624 }
625 
626 #if __TSAN_HAS_INT128
627 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_sub(volatile a128 * a,a128 v,morder mo)628 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
629   SCOPED_ATOMIC(FetchSub, a, v, mo);
630 }
631 #endif
632 
633 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_and(volatile a8 * a,a8 v,morder mo)634 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
635   SCOPED_ATOMIC(FetchAnd, a, v, mo);
636 }
637 
638 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_and(volatile a16 * a,a16 v,morder mo)639 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
640   SCOPED_ATOMIC(FetchAnd, a, v, mo);
641 }
642 
643 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_and(volatile a32 * a,a32 v,morder mo)644 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
645   SCOPED_ATOMIC(FetchAnd, a, v, mo);
646 }
647 
648 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_and(volatile a64 * a,a64 v,morder mo)649 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
650   SCOPED_ATOMIC(FetchAnd, a, v, mo);
651 }
652 
653 #if __TSAN_HAS_INT128
654 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_and(volatile a128 * a,a128 v,morder mo)655 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
656   SCOPED_ATOMIC(FetchAnd, a, v, mo);
657 }
658 #endif
659 
660 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_or(volatile a8 * a,a8 v,morder mo)661 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
662   SCOPED_ATOMIC(FetchOr, a, v, mo);
663 }
664 
665 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_or(volatile a16 * a,a16 v,morder mo)666 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
667   SCOPED_ATOMIC(FetchOr, a, v, mo);
668 }
669 
670 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_or(volatile a32 * a,a32 v,morder mo)671 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
672   SCOPED_ATOMIC(FetchOr, a, v, mo);
673 }
674 
675 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_or(volatile a64 * a,a64 v,morder mo)676 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
677   SCOPED_ATOMIC(FetchOr, a, v, mo);
678 }
679 
680 #if __TSAN_HAS_INT128
681 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_or(volatile a128 * a,a128 v,morder mo)682 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
683   SCOPED_ATOMIC(FetchOr, a, v, mo);
684 }
685 #endif
686 
687 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_xor(volatile a8 * a,a8 v,morder mo)688 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
689   SCOPED_ATOMIC(FetchXor, a, v, mo);
690 }
691 
692 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_xor(volatile a16 * a,a16 v,morder mo)693 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
694   SCOPED_ATOMIC(FetchXor, a, v, mo);
695 }
696 
697 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_xor(volatile a32 * a,a32 v,morder mo)698 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
699   SCOPED_ATOMIC(FetchXor, a, v, mo);
700 }
701 
702 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_xor(volatile a64 * a,a64 v,morder mo)703 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
704   SCOPED_ATOMIC(FetchXor, a, v, mo);
705 }
706 
707 #if __TSAN_HAS_INT128
708 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_xor(volatile a128 * a,a128 v,morder mo)709 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
710   SCOPED_ATOMIC(FetchXor, a, v, mo);
711 }
712 #endif
713 
714 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_fetch_nand(volatile a8 * a,a8 v,morder mo)715 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
716   SCOPED_ATOMIC(FetchNand, a, v, mo);
717 }
718 
719 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_fetch_nand(volatile a16 * a,a16 v,morder mo)720 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
721   SCOPED_ATOMIC(FetchNand, a, v, mo);
722 }
723 
724 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_fetch_nand(volatile a32 * a,a32 v,morder mo)725 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
726   SCOPED_ATOMIC(FetchNand, a, v, mo);
727 }
728 
729 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_fetch_nand(volatile a64 * a,a64 v,morder mo)730 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
731   SCOPED_ATOMIC(FetchNand, a, v, mo);
732 }
733 
734 #if __TSAN_HAS_INT128
735 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_fetch_nand(volatile a128 * a,a128 v,morder mo)736 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
737   SCOPED_ATOMIC(FetchNand, a, v, mo);
738 }
739 #endif
740 
741 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_compare_exchange_strong(volatile a8 * a,a8 * c,a8 v,morder mo,morder fmo)742 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
743     morder mo, morder fmo) {
744   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
745 }
746 
747 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_compare_exchange_strong(volatile a16 * a,a16 * c,a16 v,morder mo,morder fmo)748 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
749     morder mo, morder fmo) {
750   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
751 }
752 
753 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_compare_exchange_strong(volatile a32 * a,a32 * c,a32 v,morder mo,morder fmo)754 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
755     morder mo, morder fmo) {
756   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
757 }
758 
759 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_compare_exchange_strong(volatile a64 * a,a64 * c,a64 v,morder mo,morder fmo)760 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
761     morder mo, morder fmo) {
762   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
763 }
764 
765 #if __TSAN_HAS_INT128
766 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_compare_exchange_strong(volatile a128 * a,a128 * c,a128 v,morder mo,morder fmo)767 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
768     morder mo, morder fmo) {
769   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
770 }
771 #endif
772 
773 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_compare_exchange_weak(volatile a8 * a,a8 * c,a8 v,morder mo,morder fmo)774 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
775     morder mo, morder fmo) {
776   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
777 }
778 
779 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_compare_exchange_weak(volatile a16 * a,a16 * c,a16 v,morder mo,morder fmo)780 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
781     morder mo, morder fmo) {
782   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
783 }
784 
785 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_compare_exchange_weak(volatile a32 * a,a32 * c,a32 v,morder mo,morder fmo)786 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
787     morder mo, morder fmo) {
788   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
789 }
790 
791 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_compare_exchange_weak(volatile a64 * a,a64 * c,a64 v,morder mo,morder fmo)792 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
793     morder mo, morder fmo) {
794   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
795 }
796 
797 #if __TSAN_HAS_INT128
798 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_compare_exchange_weak(volatile a128 * a,a128 * c,a128 v,morder mo,morder fmo)799 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
800     morder mo, morder fmo) {
801   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
802 }
803 #endif
804 
805 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic8_compare_exchange_val(volatile a8 * a,a8 c,a8 v,morder mo,morder fmo)806 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
807     morder mo, morder fmo) {
808   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
809 }
810 
811 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic16_compare_exchange_val(volatile a16 * a,a16 c,a16 v,morder mo,morder fmo)812 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
813     morder mo, morder fmo) {
814   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
815 }
816 
817 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic32_compare_exchange_val(volatile a32 * a,a32 c,a32 v,morder mo,morder fmo)818 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
819     morder mo, morder fmo) {
820   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
821 }
822 
823 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic64_compare_exchange_val(volatile a64 * a,a64 c,a64 v,morder mo,morder fmo)824 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
825     morder mo, morder fmo) {
826   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
827 }
828 
829 #if __TSAN_HAS_INT128
830 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic128_compare_exchange_val(volatile a128 * a,a128 c,a128 v,morder mo,morder fmo)831 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
832     morder mo, morder fmo) {
833   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
834 }
835 #endif
836 
837 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic_thread_fence(morder mo)838 void __tsan_atomic_thread_fence(morder mo) {
839   char* a = 0;
840   SCOPED_ATOMIC(Fence, mo);
841 }
842 
843 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_atomic_signal_fence(morder mo)844 void __tsan_atomic_signal_fence(morder mo) {
845 }
846 }  // extern "C"
847 
848 #else  // #ifndef SANITIZER_GO
849 
850 // Go
851 
852 #define ATOMIC(func, ...) \
853     if (thr->ignore_sync) { \
854       NoTsanAtomic##func(__VA_ARGS__); \
855     } else { \
856       FuncEntry(thr, cpc); \
857       Atomic##func(thr, pc, __VA_ARGS__); \
858       FuncExit(thr); \
859     } \
860 /**/
861 
862 #define ATOMIC_RET(func, ret, ...) \
863     if (thr->ignore_sync) { \
864       (ret) = NoTsanAtomic##func(__VA_ARGS__); \
865     } else { \
866       FuncEntry(thr, cpc); \
867       (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
868       FuncExit(thr); \
869     } \
870 /**/
871 
872 extern "C" {
873 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_load(ThreadState * thr,uptr cpc,uptr pc,u8 * a)874 void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
875   ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
876 }
877 
878 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_load(ThreadState * thr,uptr cpc,uptr pc,u8 * a)879 void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
880   ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
881 }
882 
883 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_store(ThreadState * thr,uptr cpc,uptr pc,u8 * a)884 void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
885   ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
886 }
887 
888 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_store(ThreadState * thr,uptr cpc,uptr pc,u8 * a)889 void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
890   ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
891 }
892 
893 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_fetch_add(ThreadState * thr,uptr cpc,uptr pc,u8 * a)894 void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
895   ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
896 }
897 
898 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_fetch_add(ThreadState * thr,uptr cpc,uptr pc,u8 * a)899 void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
900   ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
901 }
902 
903 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)904 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
905   ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
906 }
907 
908 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)909 void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
910   ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
911 }
912 
913 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic32_compare_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)914 void __tsan_go_atomic32_compare_exchange(
915     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
916   a32 cur = 0;
917   a32 cmp = *(a32*)(a+8);
918   ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
919   *(bool*)(a+16) = (cur == cmp);
920 }
921 
922 SANITIZER_INTERFACE_ATTRIBUTE
__tsan_go_atomic64_compare_exchange(ThreadState * thr,uptr cpc,uptr pc,u8 * a)923 void __tsan_go_atomic64_compare_exchange(
924     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
925   a64 cur = 0;
926   a64 cmp = *(a64*)(a+8);
927   ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
928   *(bool*)(a+24) = (cur == cmp);
929 }
930 }  // extern "C"
931 #endif  // #ifndef SANITIZER_GO
932