1 /**
2 * hdr_atomic.h
3 * Written by Philip Orwig and released to the public domain,
4 * as explained at http://creativecommons.org/publicdomain/zero/1.0/
5 */
6
7 #ifndef HDR_ATOMIC_H__
8 #define HDR_ATOMIC_H__
9
10
11 #if defined(_MSC_VER) && !(defined(__clang__) && (defined(_M_ARM) || defined(_M_ARM64)))
12
13 #include <stdint.h>
14 #include <intrin.h>
15 #include <stdbool.h>
16
hdr_atomic_load_pointer(void ** pointer)17 static void __inline * hdr_atomic_load_pointer(void** pointer)
18 {
19 _ReadBarrier();
20 return *pointer;
21 }
22
hdr_atomic_store_pointer(void ** pointer,void * value)23 static void hdr_atomic_store_pointer(void** pointer, void* value)
24 {
25 _WriteBarrier();
26 *pointer = value;
27 }
28
hdr_atomic_load_64(int64_t * field)29 static int64_t __inline hdr_atomic_load_64(int64_t* field)
30 {
31 _ReadBarrier();
32 return *field;
33 }
34
hdr_atomic_store_64(int64_t * field,int64_t value)35 static void __inline hdr_atomic_store_64(int64_t* field, int64_t value)
36 {
37 _WriteBarrier();
38 *field = value;
39 }
40
hdr_atomic_exchange_64(volatile int64_t * field,int64_t value)41 static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t value)
42 {
43 #if defined(_WIN64)
44 return _InterlockedExchange64(field, value);
45 #else
46 int64_t comparand;
47 int64_t initial_value = *field;
48 do
49 {
50 comparand = initial_value;
51 initial_value = _InterlockedCompareExchange64(field, value, comparand);
52 }
53 while (comparand != initial_value);
54
55 return initial_value;
56 #endif
57 }
58
hdr_atomic_add_fetch_64(volatile int64_t * field,int64_t value)59 static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value)
60 {
61 #if defined(_WIN64)
62 return _InterlockedExchangeAdd64(field, value) + value;
63 #else
64 int64_t comparand;
65 int64_t initial_value = *field;
66 do
67 {
68 comparand = initial_value;
69 initial_value = _InterlockedCompareExchange64(field, comparand + value, comparand);
70 }
71 while (comparand != initial_value);
72
73 return initial_value + value;
74 #endif
75 }
76
hdr_atomic_compare_exchange_64(volatile int64_t * field,int64_t * expected,int64_t desired)77 static bool __inline hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired)
78 {
79 return *expected == _InterlockedCompareExchange64(field, desired, *expected);
80 }
81
82 #elif defined(__ATOMIC_SEQ_CST)
83
84 #define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST)
85 #define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST)
86 #define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST)
87 #define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST)
88 #define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST)
89 #define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST)
90 #define hdr_atomic_compare_exchange_64(field, expected, desired) __atomic_compare_exchange_n(field, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
91
92 #elif defined(__x86_64__)
93
94 #include <stdint.h>
95 #include <stdbool.h>
96
hdr_atomic_load_pointer(void ** pointer)97 static inline void* hdr_atomic_load_pointer(void** pointer)
98 {
99 void* p = *pointer;
100 asm volatile ("" ::: "memory");
101 return p;
102 }
103
hdr_atomic_store_pointer(void ** pointer,void * value)104 static inline void hdr_atomic_store_pointer(void** pointer, void* value)
105 {
106 asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*pointer));
107 }
108
hdr_atomic_load_64(int64_t * field)109 static inline int64_t hdr_atomic_load_64(int64_t* field)
110 {
111 int64_t i = *field;
112 asm volatile ("" ::: "memory");
113 return i;
114 }
115
hdr_atomic_store_64(int64_t * field,int64_t value)116 static inline void hdr_atomic_store_64(int64_t* field, int64_t value)
117 {
118 asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*field));
119 }
120
hdr_atomic_exchange_64(volatile int64_t * field,int64_t value)121 static inline int64_t hdr_atomic_exchange_64(volatile int64_t* field, int64_t value)
122 {
123 int64_t result = 0;
124 asm volatile ("lock; xchgq %1, %2" : "=r" (result), "+q" (value), "+m" (*field));
125 return result;
126 }
127
hdr_atomic_add_fetch_64(volatile int64_t * field,int64_t value)128 static inline int64_t hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value)
129 {
130 return __sync_add_and_fetch(field, value);
131 }
132
hdr_atomic_compare_exchange_64(volatile int64_t * field,int64_t * expected,int64_t desired)133 static inline bool hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired)
134 {
135 int64_t original;
136 asm volatile( "lock; cmpxchgq %2, %1" : "=a"(original), "+m"(*field) : "q"(desired), "0"(*expected));
137 return original == *expected;
138 }
139
140 #else
141
142 #error "Unable to determine atomic operations for your platform"
143
144 #endif
145
146 #endif /* HDR_ATOMIC_H__ */
147