81 template<typename T> T func_xchg(volatile T *v, T op) {  in func_xchg()
88 template<typename T> T func_add(volatile T *v, T op) {  in func_add()
92 template<typename T> T func_sub(volatile T *v, T op) {  in func_sub()
96 template<typename T> T func_and(volatile T *v, T op) {  in func_and()
100 template<typename T> T func_or(volatile T *v, T op) {  in func_or()
104 template<typename T> T func_xor(volatile T *v, T op) {  in func_xor()
108 template<typename T> T func_nand(volatile T *v, T op) {  in func_nand()
120 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {  in func_cas()
130 a128 func_xchg(volatile a128 *v, a128 op) {  in func_xchg()
137 a128 func_add(volatile a128 *v, a128 op) {  in func_add()
144 a128 func_sub(volatile a128 *v, a128 op) {  in func_sub()
151 a128 func_and(volatile a128 *v, a128 op) {  in func_and()
158 a128 func_or(volatile a128 *v, a128 op) {  in func_or()
165 a128 func_xor(volatile a128 *v, a128 op) {  in func_xor()
172 a128 func_nand(volatile a128 *v, a128 op) {  in func_nand()
179 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {  in func_cas()
257   T v = NoTsanAtomicLoad(a, mo);  in AtomicLoad()  local
264 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {  in NoTsanAtomicStore()
269 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {  in NoTsanAtomicStore()
276 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicStore()
299 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {  in AtomicRMW()
321 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {  in NoTsanAtomicExchange()
326 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {  in NoTsanAtomicFetchAdd()
331 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {  in NoTsanAtomicFetchSub()
336 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {  in NoTsanAtomicFetchAnd()
341 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {  in NoTsanAtomicFetchOr()
346 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {  in NoTsanAtomicFetchXor()
351 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {  in NoTsanAtomicFetchNand()
356 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicExchange()
362 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicFetchAdd()
368 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicFetchSub()
374 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicFetchAnd()
380 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicFetchOr()
386 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicFetchXor()
392 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,  in AtomicFetchNand()
398 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {  in NoTsanAtomicCAS()
403 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,  in NoTsanAtomicCAS()
415 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {  in NoTsanAtomicCAS()
422     volatile T *a, T *c, T v, morder mo, morder fmo) {  in AtomicCAS()
455     volatile T *a, T c, T v, morder mo, morder fmo) {  in AtomicCAS()
549 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_store()
554 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_store()
559 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_store()
564 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_store()
570 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_store()
576 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_exchange()
581 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_exchange()
586 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_exchange()
591 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_exchange()
597 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_exchange()
603 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_fetch_add()
608 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_fetch_add()
613 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_fetch_add()
618 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_fetch_add()
624 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_fetch_add()
630 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_fetch_sub()
635 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_fetch_sub()
640 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_fetch_sub()
645 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_fetch_sub()
651 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_fetch_sub()
657 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_fetch_and()
662 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_fetch_and()
667 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_fetch_and()
672 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_fetch_and()
678 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_fetch_and()
684 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_fetch_or()
689 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_fetch_or()
694 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_fetch_or()
699 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_fetch_or()
705 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_fetch_or()
711 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_fetch_xor()
716 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_fetch_xor()
721 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_fetch_xor()
726 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_fetch_xor()
732 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_fetch_xor()
738 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {  in __tsan_atomic8_fetch_nand()
743 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {  in __tsan_atomic16_fetch_nand()
748 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {  in __tsan_atomic32_fetch_nand()
753 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {  in __tsan_atomic64_fetch_nand()
759 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {  in __tsan_atomic128_fetch_nand()
765 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,  in __tsan_atomic8_compare_exchange_strong()
771 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,  in __tsan_atomic16_compare_exchange_strong()
777 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,  in __tsan_atomic32_compare_exchange_strong()
783 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,  in __tsan_atomic64_compare_exchange_strong()
790 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,  in __tsan_atomic128_compare_exchange_strong()
797 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,  in __tsan_atomic8_compare_exchange_weak()
803 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,  in __tsan_atomic16_compare_exchange_weak()
809 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,  in __tsan_atomic32_compare_exchange_weak()
815 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,  in __tsan_atomic64_compare_exchange_weak()
822 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,  in __tsan_atomic128_compare_exchange_weak()
829 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,  in __tsan_atomic8_compare_exchange_val()
835 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,  in __tsan_atomic16_compare_exchange_val()
841 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,  in __tsan_atomic32_compare_exchange_val()
847 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,  in __tsan_atomic64_compare_exchange_val()
854 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,  in __tsan_atomic128_compare_exchange_val()