• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef JEMALLOC_INTERNAL_ATOMIC_MSVC_H
2 #define JEMALLOC_INTERNAL_ATOMIC_MSVC_H
3 
4 #define ATOMIC_INIT(...) {__VA_ARGS__}
5 
6 typedef enum {
7 	atomic_memory_order_relaxed,
8 	atomic_memory_order_acquire,
9 	atomic_memory_order_release,
10 	atomic_memory_order_acq_rel,
11 	atomic_memory_order_seq_cst
12 } atomic_memory_order_t;
13 
14 typedef char atomic_repr_0_t;
15 typedef short atomic_repr_1_t;
16 typedef long atomic_repr_2_t;
17 typedef __int64 atomic_repr_3_t;
18 
19 ATOMIC_INLINE void
atomic_fence(atomic_memory_order_t mo)20 atomic_fence(atomic_memory_order_t mo) {
21 	_ReadWriteBarrier();
22 #  if defined(_M_ARM) || defined(_M_ARM64)
23 	/* ARM needs a barrier for everything but relaxed. */
24 	if (mo != atomic_memory_order_relaxed) {
25 		MemoryBarrier();
26 	}
27 #  elif defined(_M_IX86) || defined (_M_X64)
28 	/* x86 needs a barrier only for seq_cst. */
29 	if (mo == atomic_memory_order_seq_cst) {
30 		MemoryBarrier();
31 	}
32 #  else
33 #  error "Don't know how to create atomics for this platform for MSVC."
34 #  endif
35 	_ReadWriteBarrier();
36 }
37 
38 #define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_ ## lg_size ## _t
39 
40 #define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b)
41 #define ATOMIC_RAW_CONCAT(a, b) a ## b
42 
43 #define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) ATOMIC_CONCAT(	\
44     base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size))
45 
46 #define ATOMIC_INTERLOCKED_SUFFIX(lg_size)				\
47     ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size)
48 
49 #define ATOMIC_INTERLOCKED_SUFFIX_0 8
50 #define ATOMIC_INTERLOCKED_SUFFIX_1 16
51 #define ATOMIC_INTERLOCKED_SUFFIX_2
52 #define ATOMIC_INTERLOCKED_SUFFIX_3 64
53 
54 #define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size)		\
55 typedef struct {							\
56 	ATOMIC_INTERLOCKED_REPR(lg_size) repr;				\
57 } atomic_##short_type##_t;						\
58 									\
59 ATOMIC_INLINE type							\
60 atomic_load_##short_type(const atomic_##short_type##_t *a,		\
61     atomic_memory_order_t mo) {						\
62 	ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr;			\
63 	if (mo != atomic_memory_order_relaxed) {			\
64 		atomic_fence(atomic_memory_order_acquire);		\
65 	}								\
66 	return (type) ret;						\
67 }									\
68 									\
69 ATOMIC_INLINE void							\
70 atomic_store_##short_type(atomic_##short_type##_t *a,			\
71     type val, atomic_memory_order_t mo) {				\
72 	if (mo != atomic_memory_order_relaxed) {			\
73 		atomic_fence(atomic_memory_order_release);		\
74 	}								\
75 	a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size)) val;		\
76 	if (mo == atomic_memory_order_seq_cst) {			\
77 		atomic_fence(atomic_memory_order_seq_cst);		\
78 	}								\
79 }									\
80 									\
81 ATOMIC_INLINE type							\
82 atomic_exchange_##short_type(atomic_##short_type##_t *a, type val,	\
83     atomic_memory_order_t mo) {						\
84 	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange,	\
85 	    lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);	\
86 }									\
87 									\
88 ATOMIC_INLINE bool							\
89 atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a,	\
90     type *expected, type desired, atomic_memory_order_t success_mo,	\
91     atomic_memory_order_t failure_mo) {					\
92 	ATOMIC_INTERLOCKED_REPR(lg_size) e =				\
93 	    (ATOMIC_INTERLOCKED_REPR(lg_size))*expected;		\
94 	ATOMIC_INTERLOCKED_REPR(lg_size) d =				\
95 	    (ATOMIC_INTERLOCKED_REPR(lg_size))desired;			\
96 	ATOMIC_INTERLOCKED_REPR(lg_size) old =				\
97 	    ATOMIC_INTERLOCKED_NAME(_InterlockedCompareExchange, 	\
98 		lg_size)(&a->repr, d, e);				\
99 	if (old == e) {							\
100 		return true;						\
101 	} else {							\
102 		*expected = (type)old;					\
103 		return false;						\
104 	}								\
105 }									\
106 									\
107 ATOMIC_INLINE bool							\
108 atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a,	\
109     type *expected, type desired, atomic_memory_order_t success_mo,	\
110     atomic_memory_order_t failure_mo) {					\
111 	/* We implement the weak version with strong semantics. */	\
112 	return atomic_compare_exchange_weak_##short_type(a, expected,	\
113 	    desired, success_mo, failure_mo);				\
114 }
115 
116 
117 #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size)	\
118 JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size)			\
119 									\
120 ATOMIC_INLINE type							\
121 atomic_fetch_add_##short_type(atomic_##short_type##_t *a,		\
122     type val, atomic_memory_order_t mo) {				\
123 	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchangeAdd,	\
124 	    lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);	\
125 }									\
126 									\
127 ATOMIC_INLINE type							\
128 atomic_fetch_sub_##short_type(atomic_##short_type##_t *a,		\
129     type val, atomic_memory_order_t mo) {				\
130 	/*								\
131 	 * MSVC warns on negation of unsigned operands, but for us it	\
132 	 * gives exactly the right semantics (MAX_TYPE + 1 - operand).	\
133 	 */								\
134 	__pragma(warning(push))						\
135 	__pragma(warning(disable: 4146))				\
136 	return atomic_fetch_add_##short_type(a, -val, mo);		\
137 	__pragma(warning(pop))						\
138 }									\
139 ATOMIC_INLINE type							\
140 atomic_fetch_and_##short_type(atomic_##short_type##_t *a,		\
141     type val, atomic_memory_order_t mo) {				\
142 	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedAnd, lg_size)(	\
143 	    &a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);		\
144 }									\
145 ATOMIC_INLINE type							\
146 atomic_fetch_or_##short_type(atomic_##short_type##_t *a,		\
147     type val, atomic_memory_order_t mo) {				\
148 	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedOr, lg_size)(	\
149 	    &a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);		\
150 }									\
151 ATOMIC_INLINE type							\
152 atomic_fetch_xor_##short_type(atomic_##short_type##_t *a,		\
153     type val, atomic_memory_order_t mo) {				\
154 	return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedXor, lg_size)(	\
155 	    &a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val);		\
156 }
157 
158 #endif /* JEMALLOC_INTERNAL_ATOMIC_MSVC_H */
159