1 #ifndef _ASM_POWERPC_CMPXCHG_H_
2 #define _ASM_POWERPC_CMPXCHG_H_
3
4 #ifdef __KERNEL__
5 #include <linux/compiler.h>
6 #include <asm/synch.h>
7 #include <asm/asm-compat.h>
8
9 /*
10 * Atomic exchange
11 *
12 * Changes the memory location '*ptr' to be val and returns
13 * the previous value stored there.
14 */
15 static __always_inline unsigned long
__xchg_u32(volatile void * p,unsigned long val)16 __xchg_u32(volatile void *p, unsigned long val)
17 {
18 unsigned long prev;
19
20 __asm__ __volatile__(
21 PPC_ATOMIC_ENTRY_BARRIER
22 "1: lwarx %0,0,%2 \n"
23 PPC405_ERR77(0,%2)
24 " stwcx. %3,0,%2 \n\
25 bne- 1b"
26 PPC_ATOMIC_EXIT_BARRIER
27 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
28 : "r" (p), "r" (val)
29 : "cc", "memory");
30
31 return prev;
32 }
33
34 /*
35 * Atomic exchange
36 *
37 * Changes the memory location '*ptr' to be val and returns
38 * the previous value stored there.
39 */
40 static __always_inline unsigned long
__xchg_u32_local(volatile void * p,unsigned long val)41 __xchg_u32_local(volatile void *p, unsigned long val)
42 {
43 unsigned long prev;
44
45 __asm__ __volatile__(
46 "1: lwarx %0,0,%2 \n"
47 PPC405_ERR77(0,%2)
48 " stwcx. %3,0,%2 \n\
49 bne- 1b"
50 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
51 : "r" (p), "r" (val)
52 : "cc", "memory");
53
54 return prev;
55 }
56
57 #ifdef CONFIG_PPC64
58 static __always_inline unsigned long
__xchg_u64(volatile void * p,unsigned long val)59 __xchg_u64(volatile void *p, unsigned long val)
60 {
61 unsigned long prev;
62
63 __asm__ __volatile__(
64 PPC_ATOMIC_ENTRY_BARRIER
65 "1: ldarx %0,0,%2 \n"
66 PPC405_ERR77(0,%2)
67 " stdcx. %3,0,%2 \n\
68 bne- 1b"
69 PPC_ATOMIC_EXIT_BARRIER
70 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
71 : "r" (p), "r" (val)
72 : "cc", "memory");
73
74 return prev;
75 }
76
77 static __always_inline unsigned long
__xchg_u64_local(volatile void * p,unsigned long val)78 __xchg_u64_local(volatile void *p, unsigned long val)
79 {
80 unsigned long prev;
81
82 __asm__ __volatile__(
83 "1: ldarx %0,0,%2 \n"
84 PPC405_ERR77(0,%2)
85 " stdcx. %3,0,%2 \n\
86 bne- 1b"
87 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
88 : "r" (p), "r" (val)
89 : "cc", "memory");
90
91 return prev;
92 }
93 #endif
94
95 /*
96 * This function doesn't exist, so you'll get a linker error
97 * if something tries to do an invalid xchg().
98 */
99 extern void __xchg_called_with_bad_pointer(void);
100
101 static __always_inline unsigned long
__xchg(volatile void * ptr,unsigned long x,unsigned int size)102 __xchg(volatile void *ptr, unsigned long x, unsigned int size)
103 {
104 switch (size) {
105 case 4:
106 return __xchg_u32(ptr, x);
107 #ifdef CONFIG_PPC64
108 case 8:
109 return __xchg_u64(ptr, x);
110 #endif
111 }
112 __xchg_called_with_bad_pointer();
113 return x;
114 }
115
116 static __always_inline unsigned long
__xchg_local(volatile void * ptr,unsigned long x,unsigned int size)117 __xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
118 {
119 switch (size) {
120 case 4:
121 return __xchg_u32_local(ptr, x);
122 #ifdef CONFIG_PPC64
123 case 8:
124 return __xchg_u64_local(ptr, x);
125 #endif
126 }
127 __xchg_called_with_bad_pointer();
128 return x;
129 }
130 #define xchg(ptr,x) \
131 ({ \
132 __typeof__(*(ptr)) _x_ = (x); \
133 (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
134 })
135
136 #define xchg_local(ptr,x) \
137 ({ \
138 __typeof__(*(ptr)) _x_ = (x); \
139 (__typeof__(*(ptr))) __xchg_local((ptr), \
140 (unsigned long)_x_, sizeof(*(ptr))); \
141 })
142
143 /*
144 * Compare and exchange - if *p == old, set it to new,
145 * and return the old value of *p.
146 */
147
148 static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int * p,unsigned long old,unsigned long new)149 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
150 {
151 unsigned int prev;
152
153 __asm__ __volatile__ (
154 PPC_ATOMIC_ENTRY_BARRIER
155 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
156 cmpw 0,%0,%3\n\
157 bne- 2f\n"
158 PPC405_ERR77(0,%2)
159 " stwcx. %4,0,%2\n\
160 bne- 1b"
161 PPC_ATOMIC_EXIT_BARRIER
162 "\n\
163 2:"
164 : "=&r" (prev), "+m" (*p)
165 : "r" (p), "r" (old), "r" (new)
166 : "cc", "memory");
167
168 return prev;
169 }
170
171 static __always_inline unsigned long
__cmpxchg_u32_local(volatile unsigned int * p,unsigned long old,unsigned long new)172 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
173 unsigned long new)
174 {
175 unsigned int prev;
176
177 __asm__ __volatile__ (
178 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
179 cmpw 0,%0,%3\n\
180 bne- 2f\n"
181 PPC405_ERR77(0,%2)
182 " stwcx. %4,0,%2\n\
183 bne- 1b"
184 "\n\
185 2:"
186 : "=&r" (prev), "+m" (*p)
187 : "r" (p), "r" (old), "r" (new)
188 : "cc", "memory");
189
190 return prev;
191 }
192
193 #ifdef CONFIG_PPC64
194 static __always_inline unsigned long
__cmpxchg_u64(volatile unsigned long * p,unsigned long old,unsigned long new)195 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
196 {
197 unsigned long prev;
198
199 __asm__ __volatile__ (
200 PPC_ATOMIC_ENTRY_BARRIER
201 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
202 cmpd 0,%0,%3\n\
203 bne- 2f\n\
204 stdcx. %4,0,%2\n\
205 bne- 1b"
206 PPC_ATOMIC_EXIT_BARRIER
207 "\n\
208 2:"
209 : "=&r" (prev), "+m" (*p)
210 : "r" (p), "r" (old), "r" (new)
211 : "cc", "memory");
212
213 return prev;
214 }
215
216 static __always_inline unsigned long
__cmpxchg_u64_local(volatile unsigned long * p,unsigned long old,unsigned long new)217 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
218 unsigned long new)
219 {
220 unsigned long prev;
221
222 __asm__ __volatile__ (
223 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
224 cmpd 0,%0,%3\n\
225 bne- 2f\n\
226 stdcx. %4,0,%2\n\
227 bne- 1b"
228 "\n\
229 2:"
230 : "=&r" (prev), "+m" (*p)
231 : "r" (p), "r" (old), "r" (new)
232 : "cc", "memory");
233
234 return prev;
235 }
236 #endif
237
238 /* This function doesn't exist, so you'll get a linker error
239 if something tries to do an invalid cmpxchg(). */
240 extern void __cmpxchg_called_with_bad_pointer(void);
241
242 static __always_inline unsigned long
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,unsigned int size)243 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
244 unsigned int size)
245 {
246 switch (size) {
247 case 4:
248 return __cmpxchg_u32(ptr, old, new);
249 #ifdef CONFIG_PPC64
250 case 8:
251 return __cmpxchg_u64(ptr, old, new);
252 #endif
253 }
254 __cmpxchg_called_with_bad_pointer();
255 return old;
256 }
257
258 static __always_inline unsigned long
__cmpxchg_local(volatile void * ptr,unsigned long old,unsigned long new,unsigned int size)259 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
260 unsigned int size)
261 {
262 switch (size) {
263 case 4:
264 return __cmpxchg_u32_local(ptr, old, new);
265 #ifdef CONFIG_PPC64
266 case 8:
267 return __cmpxchg_u64_local(ptr, old, new);
268 #endif
269 }
270 __cmpxchg_called_with_bad_pointer();
271 return old;
272 }
273
274 #define cmpxchg(ptr, o, n) \
275 ({ \
276 __typeof__(*(ptr)) _o_ = (o); \
277 __typeof__(*(ptr)) _n_ = (n); \
278 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
279 (unsigned long)_n_, sizeof(*(ptr))); \
280 })
281
282
283 #define cmpxchg_local(ptr, o, n) \
284 ({ \
285 __typeof__(*(ptr)) _o_ = (o); \
286 __typeof__(*(ptr)) _n_ = (n); \
287 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
288 (unsigned long)_n_, sizeof(*(ptr))); \
289 })
290
291 #ifdef CONFIG_PPC64
292 #define cmpxchg64(ptr, o, n) \
293 ({ \
294 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
295 cmpxchg((ptr), (o), (n)); \
296 })
297 #define cmpxchg64_local(ptr, o, n) \
298 ({ \
299 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
300 cmpxchg_local((ptr), (o), (n)); \
301 })
302 #define cmpxchg64_relaxed cmpxchg64_local
303 #else
304 #include <asm-generic/cmpxchg-local.h>
305 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
306 #endif
307
308 #endif /* __KERNEL__ */
309 #endif /* _ASM_POWERPC_CMPXCHG_H_ */
310